Skip to content
Snippets Groups Projects
Commit 10abcb6f authored by Sigmund, Dominik's avatar Sigmund, Dominik
Browse files

fix testing in gitlab ci

parent e3e96441
No related branches found
No related tags found
No related merge requests found
Pipeline #82773 canceled
...@@ -8,6 +8,9 @@ import ( ...@@ -8,6 +8,9 @@ import (
"path/filepath" "path/filepath"
"datasmith/utils" "datasmith/utils"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"regexp"
"strconv"
"time"
) )
type Field struct { type Field struct {
...@@ -33,6 +36,8 @@ type DatasmithConfig struct { ...@@ -33,6 +36,8 @@ type DatasmithConfig struct {
Tables map[string]TableModel `yaml:"tables"` Tables map[string]TableModel `yaml:"tables"`
} }
func Add(tableName string, model string) { func Add(tableName string, model string) {
utils.PromptIfEmpty(&tableName, "Enter the name of the table: ") utils.PromptIfEmpty(&tableName, "Enter the name of the table: ")
...@@ -66,12 +71,8 @@ func Add(tableName string, model string) { ...@@ -66,12 +71,8 @@ func Add(tableName string, model string) {
// TODO: Do Stuff: // TODO: Do Stuff:
/* /*
- Generate Test Data for the new table if wanted - Generate Test Data for the new table if wanted
- Add Table to import-sql.sh
- Add Description and mermaid and DBML to the README.md file - Add Description and mermaid and DBML to the README.md file
- Add Test to gitlab-ci.yml - Fix prompting to use SelectMenu
- Add to CHANGELOG.md
- Bump version in datasmith.yaml
*/ */
slug := utils.Slugify(tableName) slug := utils.Slugify(tableName)
...@@ -82,6 +83,17 @@ func Add(tableName string, model string) { ...@@ -82,6 +83,17 @@ func Add(tableName string, model string) {
config.Tables = make(map[string]TableModel) config.Tables = make(map[string]TableModel)
} }
config.Tables[slug] = tableModel config.Tables[slug] = tableModel
// Bump the version
newVersion, err := bumpVersion(config.Version)
if err != nil {
fmt.Printf("Error bumping version: %v\n", err)
return
} else {
fmt.Printf("Bumped version to %s\n", newVersion)
}
config.Version = newVersion
if err := saveConfig(projectDir, config); err != nil { if err := saveConfig(projectDir, config); err != nil {
fmt.Printf("Error saving config: %v\n", err) fmt.Printf("Error saving config: %v\n", err)
} }
...@@ -89,6 +101,13 @@ func Add(tableName string, model string) { ...@@ -89,6 +101,13 @@ func Add(tableName string, model string) {
// Create the table file under sql directory // Create the table file under sql directory
createTableFile(projectDir, tableName, config.DbType, tableModel) createTableFile(projectDir, tableName, config.DbType, tableModel)
// Append import command to import-sql.sh
if err := appendToImportScript(projectDir, tableName, config.DbType); err != nil {
fmt.Printf("Error appending to import-sql.sh: %v\n", err)
} else {
fmt.Printf("Appended import command for table '%s' to import-sql.sh\n", tableName)
}
// Append table description to database.dbml // Append table description to database.dbml
if err := appendToDBMLFile(projectDir, tableName, tableModel); err != nil { if err := appendToDBMLFile(projectDir, tableName, tableModel); err != nil {
fmt.Printf("Error appending to database.dbml: %v\n", err) fmt.Printf("Error appending to database.dbml: %v\n", err)
...@@ -96,6 +115,13 @@ func Add(tableName string, model string) { ...@@ -96,6 +115,13 @@ func Add(tableName string, model string) {
fmt.Printf("Appended table '%s' to database.dbml\n", tableName) fmt.Printf("Appended table '%s' to database.dbml\n", tableName)
} }
// Update CHANGELOG
if err := updateChangelog(projectDir, newVersion, tableName); err != nil {
fmt.Printf("Error updating CHANGELOG: %v\n", err)
} else {
fmt.Printf("Updated CHANGELOG with version '%s'\n", newVersion)
}
fmt.Printf("Added new table '%s' to the project\n", slug) fmt.Printf("Added new table '%s' to the project\n", slug)
} }
...@@ -294,6 +320,67 @@ func appendToDBMLFile(projectDir, tableName string, tableModel TableModel) error ...@@ -294,6 +320,67 @@ func appendToDBMLFile(projectDir, tableName string, tableModel TableModel) error
return nil return nil
} }
func updateChangelog(projectDir, newVersion, tableName string) error {
changelogFilePath := filepath.Join(projectDir, "CHANGELOG.md")
// Read the existing CHANGELOG content
file, err := os.ReadFile(changelogFilePath)
if err != nil {
return fmt.Errorf("error reading CHANGELOG.md file: %v", err)
}
content := string(file)
// Create the new changelog entry
newEntry := fmt.Sprintf("## [%s] - %s\n\n### Added\n\n- Added table `%s`\n\n", newVersion, time.Now().Format("2006-01-02"), tableName)
// Insert the new entry after the "## [Unreleased]" line
insertPos := strings.Index(content, "## [Unreleased]")
if insertPos == -1 {
// If "## [Unreleased]" not found, prepend the new entry
content = newEntry + content
} else {
insertPos += len("## [Unreleased]\n\n")
content = content[:insertPos] + newEntry + content[insertPos:]
}
// Write the updated content back to the CHANGELOG file
if err := os.WriteFile(changelogFilePath, []byte(content), 0644); err != nil {
return fmt.Errorf("error writing to CHANGELOG.md file: %v", err)
}
return nil
}
func appendToImportScript(projectDir, tableName, dbType string) error {
scriptFilePath := filepath.Join(projectDir, "import-sql.sh")
tableSQLFile := fmt.Sprintf("/tmp/%s.sql", utils.Slugify(tableName))
var importCommand string
switch dbType {
case "mysql":
importCommand = "\n" + fmt.Sprintf(`echo "Importing table %s"`, tableName) + "\n" +
fmt.Sprintf(`mariadb -u root -p"$MARIADB_ROOT_PASSWORD" "$DB_DATABASE" < %s`, tableSQLFile) + "\n"
case "postgres":
importCommand = "\n" + fmt.Sprintf(`echo "Importing table %s"`, tableName) + "\n" +
fmt.Sprintf(`psql -U "$DB_USER" -d "$DB_DATABASE" -f %s`, tableSQLFile) + "\n"
default:
return fmt.Errorf("unsupported database type: %s", dbType)
}
file, err := os.OpenFile(scriptFilePath, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
if err != nil {
return fmt.Errorf("error opening import-sql.sh file: %v", err)
}
defer file.Close()
if _, err := file.WriteString(importCommand); err != nil {
return fmt.Errorf("error writing to import-sql.sh file: %v", err)
}
return nil
}
func validateModel(model TableModel) error { func validateModel(model TableModel) error {
fieldNames := make(map[string]bool) fieldNames := make(map[string]bool)
...@@ -386,6 +473,23 @@ func promptForFieldType() string { ...@@ -386,6 +473,23 @@ func promptForFieldType() string {
return selected return selected
} }
// Function to bump the version by a minor version
func bumpVersion(version string) (string, error) {
re := regexp.MustCompile(`^(\d+)\.(\d+)\.(\d+)$`)
matches := re.FindStringSubmatch(version)
if matches == nil {
return "", fmt.Errorf("invalid version format: %s", version)
}
major, _ := strconv.Atoi(matches[1])
minor, _ := strconv.Atoi(matches[2])
patch, _ := strconv.Atoi(matches[3])
minor++
newVersion := fmt.Sprintf("%d.%d.%d", major, minor, patch)
return newVersion, nil
}
// Help returns the help information for the add command // Help returns the help information for the add command
func AddHelp() string { func AddHelp() string {
return "add <tablename> [--model <json_model>]: Add a table to the project." return "add <tablename> [--model <json_model>]: Add a table to the project."
......
...@@ -64,6 +64,9 @@ func InitProject(name string, dbType string) { ...@@ -64,6 +64,9 @@ func InitProject(name string, dbType string) {
// Create .gitlab-ci.yml file from template based on dbType // Create .gitlab-ci.yml file from template based on dbType
createGitlabCiFile(slug, dbType) createGitlabCiFile(slug, dbType)
// Create generate_db_tests.sh script from template
createGenerateDbTestsScript(slug)
// Create sqlfluff file from template // Create sqlfluff file from template
createSqlfluffFile(slug) createSqlfluffFile(slug)
...@@ -252,6 +255,16 @@ func createGitlabCiFile(projectDir, dbType string) { ...@@ -252,6 +255,16 @@ func createGitlabCiFile(projectDir, dbType string) {
} }
} }
// createGenerateDbTestsScript creates the generate_db_tests.sh script from template
func createGenerateDbTestsScript(projectDir string) {
err := templates.CreateFileFromTemplate(projectDir, "", "generate_db_tests.sh", "generate_db_tests.sh", nil)
if err != nil {
fmt.Printf("Error creating generate_db_tests.sh script: %v\n", err)
} else {
fmt.Printf("Created file: %s/generate_db_tests.sh\n", projectDir)
}
}
// createSqlfluffFile creates the sqlfluff configuration file from template // createSqlfluffFile creates the sqlfluff configuration file from template
func createSqlfluffFile(projectDir string) { func createSqlfluffFile(projectDir string) {
err := templates.CreateFileFromTemplate(projectDir, "", "sqlfluff", "sqlfluff", nil) err := templates.CreateFileFromTemplate(projectDir, "", "sqlfluff", "sqlfluff", nil)
......
...@@ -24,13 +24,19 @@ func main() { ...@@ -24,13 +24,19 @@ func main() {
case "init": case "init":
var name, dbType string var name, dbType string
args := os.Args[2:] args := os.Args[2:]
for i, arg := range args { for i := 0; i < len(args); i++ {
arg := args[i]
if arg == "--type" && i+1 < len(args) { if arg == "--type" && i+1 < len(args) {
dbType = args[i+1] dbType = args[i+1]
i++ // Skip the next argument since it's the model value
} else if !strings.HasPrefix(arg, "--") { } else if !strings.HasPrefix(arg, "--") {
name = arg name = arg
} }
} }
if name == "" {
fmt.Println("Usage: ds init <name> [--type <json_model>]")
return
}
commands.InitProject(name, dbType) commands.InitProject(name, dbType)
fmt.Println("Done. Now run 'ds add' to add a table to the database.") fmt.Println("Done. Now run 'ds add' to add a table to the database.")
case "add": case "add":
......
#!/bin/bash
# Path to the datasmith.yaml file
DATASMITH_FILE="datasmith.yaml"
# Check if the file exists
if [ ! -f "$DATASMITH_FILE" ]; then
echo "datasmith.yaml not found!"
exit 1
fi
# Read the database type from the datasmith.yaml
DB_TYPE=$(grep 'database_type:' "$DATASMITH_FILE" | awk '{print $2}')
# Read the tables from the datasmith.yaml and generate SQL commands
TABLES=$(grep -oP '^\s{2}[a-zA-Z_0-9]+(?=:)' "$DATASMITH_FILE")
if [ -z "$TABLES" ]; then
echo "No tables found in datasmith.yaml"
exit 1
fi
# Generate the appropriate SQL commands based on the database type
case $DB_TYPE in
mysql)
for TABLE in $TABLES; do
echo "mysql -h 'bb' --port 3306 -u \$DB_USER -p\$DB_PASSWORD -D \$DB_DATABASE -e 'SELECT * FROM $TABLE;'"
done
;;
postgres)
for TABLE in $TABLES; do
echo "psql -h 'bb' --port 5432 -U \$DB_USER -d \$DB_DATABASE -c 'SELECT * FROM $TABLE;'"
done
;;
*)
echo "Unsupported database type: $DB_TYPE"
exit 1
;;
esac
\ No newline at end of file
...@@ -61,7 +61,6 @@ build: ...@@ -61,7 +61,6 @@ build:
variables: variables:
FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
BUILD_ARGS:
script: script:
- echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} - echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
- docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} . - docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} .
...@@ -116,8 +115,9 @@ test:database: ...@@ -116,8 +115,9 @@ test:database:
alias: db alias: db
before_script: before_script:
- apt-get update && apt-get install -y default-mysql-client # Install MySQL client - apt-get update && apt-get install -y default-mysql-client # Install MySQL client
- chmod +x generate_db_tests.sh
script: script:
- echo "Not Tables to Test" - ./generate_db_tests.sh | bash
rules: rules:
- if: $CI_COMMIT_BRANCH == "develop" - if: $CI_COMMIT_BRANCH == "develop"
...@@ -140,7 +140,7 @@ test:dast: ...@@ -140,7 +140,7 @@ test:dast:
publish:latest: publish:latest:
stage: publish stage: publish
image: image: docker:latest image: docker:latest
needs: needs:
- test:database - test:database
- test:sast - test:sast
......
...@@ -61,7 +61,6 @@ build: ...@@ -61,7 +61,6 @@ build:
variables: variables:
FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
BUILD_ARGS:
script: script:
- echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} - echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
- docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} . - docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} .
...@@ -116,8 +115,9 @@ test:database: ...@@ -116,8 +115,9 @@ test:database:
alias: db alias: db
before_script: before_script:
- apt-get update && apt-get install -y postgresql-client # Install PostgreSQL client - apt-get update && apt-get install -y postgresql-client # Install PostgreSQL client
- chmod +x generate_db_tests.sh
script: script:
- echo "Not Tables to Test" - ./generate_db_tests.sh | bash
rules: rules:
- if: $CI_COMMIT_BRANCH == "develop" - if: $CI_COMMIT_BRANCH == "develop"
...@@ -140,7 +140,7 @@ test:dast: ...@@ -140,7 +140,7 @@ test:dast:
publish:latest: publish:latest:
stage: publish stage: publish
image: image: docker:latest image: docker:latest
needs: needs:
- test:database - test:database
- test:sast - test:sast
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment