diff --git a/commands/add.go b/commands/add.go index a0022ed3257bf4f706a20f91fd0dcd1876fd6dfe..5fd1bfa3bc2ee2b1e989a28c225c66ad978dd64b 100644 --- a/commands/add.go +++ b/commands/add.go @@ -8,6 +8,9 @@ import ( "path/filepath" "datasmith/utils" "gopkg.in/yaml.v2" + "regexp" + "strconv" + "time" ) type Field struct { @@ -33,6 +36,8 @@ type DatasmithConfig struct { Tables map[string]TableModel `yaml:"tables"` } + + func Add(tableName string, model string) { utils.PromptIfEmpty(&tableName, "Enter the name of the table: ") @@ -66,12 +71,8 @@ func Add(tableName string, model string) { // TODO: Do Stuff: /* - Generate Test Data for the new table if wanted - - Add Table to import-sql.sh - Add Description and mermaid and DBML to the README.md file - - Add Test to gitlab-ci.yml - - Add to CHANGELOG.md - - Bump version in datasmith.yaml - + - Fix prompting to use SelectMenu */ slug := utils.Slugify(tableName) @@ -82,6 +83,17 @@ func Add(tableName string, model string) { config.Tables = make(map[string]TableModel) } config.Tables[slug] = tableModel + + // Bump the version + newVersion, err := bumpVersion(config.Version) + if err != nil { + fmt.Printf("Error bumping version: %v\n", err) + return + } else { + fmt.Printf("Bumped version to %s\n", newVersion) + } + config.Version = newVersion + if err := saveConfig(projectDir, config); err != nil { fmt.Printf("Error saving config: %v\n", err) } @@ -89,6 +101,13 @@ func Add(tableName string, model string) { // Create the table file under sql directory createTableFile(projectDir, tableName, config.DbType, tableModel) + // Append import command to import-sql.sh + if err := appendToImportScript(projectDir, tableName, config.DbType); err != nil { + fmt.Printf("Error appending to import-sql.sh: %v\n", err) + } else { + fmt.Printf("Appended import command for table '%s' to import-sql.sh\n", tableName) + } + // Append table description to database.dbml if err := appendToDBMLFile(projectDir, tableName, tableModel); err != nil { fmt.Printf("Error appending to database.dbml: %v\n", err) @@ -96,6 +115,13 @@ func Add(tableName string, model string) { fmt.Printf("Appended table '%s' to database.dbml\n", tableName) } + // Update CHANGELOG + if err := updateChangelog(projectDir, newVersion, tableName); err != nil { + fmt.Printf("Error updating CHANGELOG: %v\n", err) + } else { + fmt.Printf("Updated CHANGELOG with version '%s'\n", newVersion) + } + fmt.Printf("Added new table '%s' to the project\n", slug) } @@ -294,6 +320,67 @@ func appendToDBMLFile(projectDir, tableName string, tableModel TableModel) error return nil } +func updateChangelog(projectDir, newVersion, tableName string) error { + changelogFilePath := filepath.Join(projectDir, "CHANGELOG.md") + + // Read the existing CHANGELOG content + file, err := os.ReadFile(changelogFilePath) + if err != nil { + return fmt.Errorf("error reading CHANGELOG.md file: %v", err) + } + + content := string(file) + + // Create the new changelog entry + newEntry := fmt.Sprintf("## [%s] - %s\n\n### Added\n\n- Added table `%s`\n\n", newVersion, time.Now().Format("2006-01-02"), tableName) + + // Insert the new entry after the "## [Unreleased]" line + insertPos := strings.Index(content, "## [Unreleased]") + if insertPos == -1 { + // If "## [Unreleased]" not found, prepend the new entry + content = newEntry + content + } else { + insertPos += len("## [Unreleased]\n\n") + content = content[:insertPos] + newEntry + content[insertPos:] + } + + // Write the updated content back to the CHANGELOG file + if err := os.WriteFile(changelogFilePath, []byte(content), 0644); err != nil { + return fmt.Errorf("error writing to CHANGELOG.md file: %v", err) + } + + return nil +} + +func appendToImportScript(projectDir, tableName, dbType string) error { + scriptFilePath := filepath.Join(projectDir, "import-sql.sh") + tableSQLFile := fmt.Sprintf("/tmp/%s.sql", utils.Slugify(tableName)) + + var importCommand string + switch dbType { + case "mysql": + importCommand = "\n" + fmt.Sprintf(`echo "Importing table %s"`, tableName) + "\n" + + fmt.Sprintf(`mariadb -u root -p"$MARIADB_ROOT_PASSWORD" "$DB_DATABASE" < %s`, tableSQLFile) + "\n" + case "postgres": + importCommand = "\n" + fmt.Sprintf(`echo "Importing table %s"`, tableName) + "\n" + + fmt.Sprintf(`psql -U "$DB_USER" -d "$DB_DATABASE" -f %s`, tableSQLFile) + "\n" + default: + return fmt.Errorf("unsupported database type: %s", dbType) + } + + file, err := os.OpenFile(scriptFilePath, os.O_APPEND|os.O_WRONLY, os.ModeAppend) + if err != nil { + return fmt.Errorf("error opening import-sql.sh file: %v", err) + } + defer file.Close() + + if _, err := file.WriteString(importCommand); err != nil { + return fmt.Errorf("error writing to import-sql.sh file: %v", err) + } + + return nil +} + func validateModel(model TableModel) error { fieldNames := make(map[string]bool) @@ -386,6 +473,23 @@ func promptForFieldType() string { return selected } +// Function to bump the version by a minor version +func bumpVersion(version string) (string, error) { + re := regexp.MustCompile(`^(\d+)\.(\d+)\.(\d+)$`) + matches := re.FindStringSubmatch(version) + if matches == nil { + return "", fmt.Errorf("invalid version format: %s", version) + } + + major, _ := strconv.Atoi(matches[1]) + minor, _ := strconv.Atoi(matches[2]) + patch, _ := strconv.Atoi(matches[3]) + + minor++ + newVersion := fmt.Sprintf("%d.%d.%d", major, minor, patch) + return newVersion, nil +} + // Help returns the help information for the add command func AddHelp() string { return "add <tablename> [--model <json_model>]: Add a table to the project." diff --git a/commands/init.go b/commands/init.go index 73859c25c2c5afccb0659908db96c27125842eb7..6ae586ddd3a1cbe7ca2beea1a4cc51d13dda47a0 100644 --- a/commands/init.go +++ b/commands/init.go @@ -64,6 +64,9 @@ func InitProject(name string, dbType string) { // Create .gitlab-ci.yml file from template based on dbType createGitlabCiFile(slug, dbType) + // Create generate_db_tests.sh script from template + createGenerateDbTestsScript(slug) + // Create sqlfluff file from template createSqlfluffFile(slug) @@ -252,6 +255,16 @@ func createGitlabCiFile(projectDir, dbType string) { } } +// createGenerateDbTestsScript creates the generate_db_tests.sh script from template +func createGenerateDbTestsScript(projectDir string) { + err := templates.CreateFileFromTemplate(projectDir, "", "generate_db_tests.sh", "generate_db_tests.sh", nil) + if err != nil { + fmt.Printf("Error creating generate_db_tests.sh script: %v\n", err) + } else { + fmt.Printf("Created file: %s/generate_db_tests.sh\n", projectDir) + } +} + // createSqlfluffFile creates the sqlfluff configuration file from template func createSqlfluffFile(projectDir string) { err := templates.CreateFileFromTemplate(projectDir, "", "sqlfluff", "sqlfluff", nil) diff --git a/main.go b/main.go index c5804594c519bfa8fb9c44d70d860a12f28bc1ba..8ed57fb705e335623cba96f4ec50579b3533caa9 100644 --- a/main.go +++ b/main.go @@ -24,13 +24,19 @@ func main() { case "init": var name, dbType string args := os.Args[2:] - for i, arg := range args { + for i := 0; i < len(args); i++ { + arg := args[i] if arg == "--type" && i+1 < len(args) { dbType = args[i+1] + i++ // Skip the next argument since it's the model value } else if !strings.HasPrefix(arg, "--") { name = arg } } + if name == "" { + fmt.Println("Usage: ds init <name> [--type <json_model>]") + return + } commands.InitProject(name, dbType) fmt.Println("Done. Now run 'ds add' to add a table to the database.") case "add": diff --git a/templates/generate_db_tests.sh.tmpl b/templates/generate_db_tests.sh.tmpl new file mode 100644 index 0000000000000000000000000000000000000000..0b3f07b984d740e768db5cf2501a7612ac0805dd --- /dev/null +++ b/templates/generate_db_tests.sh.tmpl @@ -0,0 +1,39 @@ +#!/bin/bash + +# Path to the datasmith.yaml file +DATASMITH_FILE="datasmith.yaml" + +# Check if the file exists +if [ ! -f "$DATASMITH_FILE" ]; then + echo "datasmith.yaml not found!" + exit 1 +fi + +# Read the database type from the datasmith.yaml +DB_TYPE=$(grep 'database_type:' "$DATASMITH_FILE" | awk '{print $2}') + +# Read the tables from the datasmith.yaml and generate SQL commands +TABLES=$(grep -oP '^\s{2}[a-zA-Z_0-9]+(?=:)' "$DATASMITH_FILE") + +if [ -z "$TABLES" ]; then + echo "No tables found in datasmith.yaml" + exit 1 +fi + +# Generate the appropriate SQL commands based on the database type +case $DB_TYPE in + mysql) + for TABLE in $TABLES; do + echo "mysql -h 'bb' --port 3306 -u \$DB_USER -p\$DB_PASSWORD -D \$DB_DATABASE -e 'SELECT * FROM $TABLE;'" + done + ;; + postgres) + for TABLE in $TABLES; do + echo "psql -h 'bb' --port 5432 -U \$DB_USER -d \$DB_DATABASE -c 'SELECT * FROM $TABLE;'" + done + ;; + *) + echo "Unsupported database type: $DB_TYPE" + exit 1 + ;; +esac \ No newline at end of file diff --git a/templates/gitlab-ci.mysql.yaml.tmpl b/templates/gitlab-ci.mysql.yaml.tmpl index ad266d28239980376dea951653849ac37f8bd8db..160ce92060ddb3ca9d82f9848c34428ebc774c6a 100644 --- a/templates/gitlab-ci.mysql.yaml.tmpl +++ b/templates/gitlab-ci.mysql.yaml.tmpl @@ -61,7 +61,6 @@ build: variables: FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar - BUILD_ARGS: script: - echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} - docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} . @@ -116,8 +115,9 @@ test:database: alias: db before_script: - apt-get update && apt-get install -y default-mysql-client # Install MySQL client + - chmod +x generate_db_tests.sh script: - - echo "Not Tables to Test" + - ./generate_db_tests.sh | bash rules: - if: $CI_COMMIT_BRANCH == "develop" @@ -140,7 +140,7 @@ test:dast: publish:latest: stage: publish - image: image: docker:latest + image: docker:latest needs: - test:database - test:sast diff --git a/templates/gitlab-ci.postgres.yaml.tmpl b/templates/gitlab-ci.postgres.yaml.tmpl index 80174002ab32a4f288d3951f4064a847dbc9cb74..a1aac1ca4c500e0fc514dc4d2aa48c88b0cc9764 100644 --- a/templates/gitlab-ci.postgres.yaml.tmpl +++ b/templates/gitlab-ci.postgres.yaml.tmpl @@ -61,7 +61,6 @@ build: variables: FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar - BUILD_ARGS: script: - echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} - docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} . @@ -116,8 +115,9 @@ test:database: alias: db before_script: - apt-get update && apt-get install -y postgresql-client # Install PostgreSQL client + - chmod +x generate_db_tests.sh script: - - echo "Not Tables to Test" + - ./generate_db_tests.sh | bash rules: - if: $CI_COMMIT_BRANCH == "develop" @@ -140,7 +140,7 @@ test:dast: publish:latest: stage: publish - image: image: docker:latest + image: docker:latest needs: - test:database - test:sast