Skip to content
Snippets Groups Projects
Commit 9a0ab451 authored by Sigmund, Dominik's avatar Sigmund, Dominik
Browse files

Added almost all templates

parent 4c10d32b
No related branches found
No related tags found
No related merge requests found
Pipeline #82247 failed
......@@ -19,7 +19,7 @@ func addTable(name string) {
- if nothing, ask for fields (name, type, primary key, foreign key, unique, not null, auto increment, default value)
- Check if the table already exists in the datasmith.yaml file
- Add a new table and columns to the datasmith.yaml file
- Create a new table file in the sql directory (by database type)
- Create a new table file in the sql directory (by database type) sql/slug.sql
- Generate Test Data for the new table if wanted
- Add Table to import-sql.sh
- Add Description to the DBML file
......
......@@ -43,12 +43,12 @@ func InitProject(name string, dbType string) {
// Create an empty database.dbml file
createDBMLFile(slug)
// Create an empty sql/database.sql file
createDatabaseSQLFile(slug)
// Create LICENSE.md file from template
createLicenseFile(slug, name)
// Create CONTRIBUTING.md file from template
createContributingFile(slug)
// Create .gitignore file from template
createGitignoreFile(slug)
......@@ -58,9 +58,14 @@ func InitProject(name string, dbType string) {
// Create Containerfile from template based on dbType
createContainerfile(slug, dbType)
// Create .gitlab-ci.yml file from template based on dbType
createGitlabCiFile(slug, dbType)
// Create sqlfluff file from template
createSqlfluffFile(slug)
// Create import-sql.sh script from template based on dbType
createImportSqlScript(slug, dbType)
// Create k8s/persistentvolumeclaim.template.yaml file from template
createK8sPersistentVolumeFile(slug)
......@@ -77,9 +82,7 @@ func InitProject(name string, dbType string) {
// TODO: copy base files from https://gitlab.ard.de/br/buzzboard/database/-/tree/develop?ref_type=heads
/*
- README.md
- CONTRIBUTING.md
- import-sql.sh (by database type)
- .gilab-ci.yml (by database type) (also import tests from blessing of the day)
- Add Tests from segen des tages
*/
fmt.Printf("Initialized new project structure in '%s'\n", slug)
......@@ -162,19 +165,6 @@ func createDBMLFile(projectDir string) {
fmt.Printf("Created file: %s/database.dbml\n", projectDir)
}
// createDatabaseSQLFile creates an empty sql/database.sql file in the project directory
func createDatabaseSQLFile(projectDir string) {
sqlFilePath := filepath.Join(projectDir, "sql", "database.sql")
file, err := os.Create(sqlFilePath)
if err != nil {
fmt.Printf("Error creating sql/database.sql file: %v\n", err)
return
}
defer file.Close()
fmt.Printf("Created file: %s/sql/database.sql\n", projectDir)
}
// createLicenseFile creates the LICENSE.md file from template
func createLicenseFile(projectDir, projectName string) {
data := map[string]interface{}{
......@@ -190,6 +180,16 @@ func createLicenseFile(projectDir, projectName string) {
}
}
// createContributingFile creates the CONTRIBUTING.md file from template
func createContributingFile(projectDir string) {
err := templates.CreateFileFromTemplate(projectDir, "", "CONTRIBUTING.md", "CONTRIBUTING.md", nil)
if err != nil {
fmt.Printf("Error creating CONTRIBUTING.md file: %v\n", err)
} else {
fmt.Printf("Created file: %s/CONTRIBUTING.md\n", projectDir)
}
}
// createGitignoreFile creates the .gitignore file from template
func createGitignoreFile(projectDir string) {
err := templates.CreateFileFromTemplate(projectDir, "", ".gitignore", "gitignore", nil)
......@@ -225,6 +225,21 @@ func createContainerfile(projectDir, dbType string) {
}
}
// createGitlabCiFile creates the .gitlab-ci.yml file from the appropriate template
func createGitlabCiFile(projectDir, dbType string) {
templateFile := "gitlab-ci.mysql.yaml"
if dbType == "postgres" {
templateFile = "gitlab-ci.postgres.yaml"
}
err := templates.CreateFileFromTemplate(projectDir, "", ".gitlab-ci.yml", templateFile, nil)
if err != nil {
fmt.Printf("Error creating .gitlab-ci.yml file: %v\n", err)
} else {
fmt.Printf("Created file: %s/.gitlab-ci.yml\n", projectDir)
}
}
// createSqlfluffFile creates the sqlfluff configuration file from template
func createSqlfluffFile(projectDir string) {
err := templates.CreateFileFromTemplate(projectDir, "", "sqlfluff", "sqlfluff", nil)
......@@ -235,6 +250,21 @@ func createSqlfluffFile(projectDir string) {
}
}
// createImportSqlScript creates the import-sql.sh script from the appropriate template
func createImportSqlScript(projectDir, dbType string) {
templateFile := "import-sql.mysql"
if dbType == "postgres" {
templateFile = "import-sql.postgres"
}
err := templates.CreateFileFromTemplate(projectDir, "", "import-sql.sh", templateFile, nil)
if err != nil {
fmt.Printf("Error creating import-sql.sh script: %v\n", err)
} else {
fmt.Printf("Created file: %s/import-sql.sh\n", projectDir)
}
}
// createK8sPersistentVolumeClaimFile creates the k8s/persistentVolume.template.yaml file from template
func createK8sPersistentVolumeFile(projectDir string) {
......
# CONTRIBUTING
We appreciate and welcome contributions from the community to enhance the features and overall quality of DataSmith. Whether you're a developer, tester, or enthusiastic user, there are several ways you can contribute:
## Creating Issues
If you encounter a bug, have a feature request, or want to suggest improvements, please [create an issue]( https://gitlab.ard.de/apps/datasmith/-/issues/new) on our Gitlab repository. When creating an issue, provide detailed information about the problem or enhancement you're addressing. This includes steps to reproduce the issue and any relevant context that can help our team understand and address it effectively.
## Pull Requests
If you'd like to contribute code, documentation, or fixes, we encourage you to submit a pull request. Before creating a pull request, please:
1. Fork the repository.
2. Create a new branch for your changes.
3. Make your modifications, ensuring adherence to our coding standards.
4. Write tests for new features or modifications.
5. Ensure all tests pass.
6. Update the datasmith.yaml file
7. Update the CHANGELOG.md file
8. Update this README if necessary
9. Submit a merge request to the `main` branch of the repository.
We'll review your pull request, provide feedback, and work with you to ensure that your contribution aligns with the project's goals and standards.
\ No newline at end of file
stages:
- lint # Lint MySQL-Files and SAST
- build # Create Container
- scan # Scan Container and push commit
- test # DAST and Acceptance against running container
- publish # publish tages latest or stable
- deploy # To k8s
workflow:
rules:
- if: $CI_COMMIT_REF_NAME == "main" # Main branch
variables:
K8S_CLUSTER: ${K8S_CLUSTER_LIVE}
K8S_CLUSTER_SERVER: ${K8S_CLUSTER_SERVER_LIVE}
K8S_CERTIFICATE_AUTHORITY_DATA: ${K8S_CERTIFICATE_AUTHORITY_DATA_LIVE}
K8S_TOKEN: ${K8S_TOKEN_LIVE}
TAG: stable
TLS_CERT: ${TLS_CERT_LIVE}
TLS_KEY: ${TLS_KEY_LIVE}
ENVIRONMENT_NAME: live
DNS_BACKEND: ${DNS_BACKEND_STAGE}
- if: $CI_COMMIT_REF_NAME == "develop" # Develop branch
variables:
K8S_CLUSTER: ${K8S_CLUSTER_STAGE}
K8S_CLUSTER_SERVER: ${K8S_CLUSTER_SERVER_STAGE}
K8S_CERTIFICATE_AUTHORITY_DATA: ${K8S_CERTIFICATE_AUTHORITY_DATA_STAGE}
K8S_TOKEN: ${K8S_TOKEN_STAGE}
TAG: latest
TLS_CERT: ${TLS_CERT_STAGE}
TLS_KEY: ${TLS_KEY_STAGE}
ENVIRONMENT_NAME: stage
DNS_BACKEND: ${DNS_BACKEND_STAGE}
- if: $CI_COMMIT_REF_NAME != "main" && $CI_COMMIT_REF_NAME != "develop" # Feature / Hotfix branch
variables:
TAG: $CI_COMMIT_SHORT_SHA
.prep-k8s: &prep-k8s # k8s connection settings
before_script:
- kubectl config set-cluster ${K8S_CLUSTER} --server="${K8S_CLUSTER_SERVER}"
- kubectl config set clusters.${K8S_CLUSTER}.certificate-authority-data ${K8S_CERTIFICATE_AUTHORITY_DATA}
- kubectl config set-credentials ${K8S_USER} --token="${K8S_TOKEN}"
- kubectl config set-context k8s --cluster=${K8S_CLUSTER} --user=${K8S_USER} --namespace=${K8S_NAMESPACE}
- kubectl config use-context k8s
# --- ALL branches automatically LINT/SAST ---#
lint:sast:
stage: lint
image: python:3.9-slim
before_script:
- pip install sqlfluff
script:
- sqlfluff lint ./sql/*.sql --dialect mysql
# --- Only Develop Builds and tests --- #
build:
stage: build
image: docker:latest
variables:
FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
BUILD_ARGS:
script:
- echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
- docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} .
- docker save ${FULL_IMAGE_NAME} -o ${OCI_IMAGE_NAME}
artifacts:
paths:
- ${OCI_IMAGE_NAME}
rules:
- if: $CI_COMMIT_BRANCH == "develop"
test:container-scan:
stage: scan
needs:
- job: build
artifacts: true
image:
name: docker.io/aquasec/trivy:latest
entrypoint: [""]
variables:
GIT_STRATEGY: none
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
TRIVY_INSECURE: "true"
script:
- trivy --version
- time trivy image --exit-code 0 --severity HIGH --no-progress --input ${OCI_IMAGE_NAME}
- time trivy image --exit-code 1 --ignore-unfixed --severity CRITICAL --no-progress --input ${OCI_IMAGE_NAME}
rules:
- if: $CI_COMMIT_BRANCH == "develop"
publish:commit:
stage: scan
image: docker:latest
rules:
- if: $CI_COMMIT_BRANCH == "develop"
needs:
- job: build
artifacts: true
- job: test:container-scan
variables:
FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
script:
- echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
- docker load -i ${OCI_IMAGE_NAME}
- docker push ${FULL_IMAGE_NAME}
test:database:
stage: test
image: mysql:latest
services:
- name: $CI_REGISTRY_IMAGE:$TAG
alias: db
before_script:
- apt-get update && apt-get install -y default-mysql-client # Install MySQL client
script:
- echo "Not Tables to Test"
rules:
- if: $CI_COMMIT_BRANCH == "develop"
test:dast:
stage: test
image: alpine:latest
services:
- name: $CI_REGISTRY_IMAGE:$TAG
alias: db
before_script:
- apk add --no-cache python3 py3-pip python3-dev build-base libffi-dev # Install required packages
script:
- python3 -m venv venv # Create a virtual environment
- source venv/bin/activate # Activate the virtual environment
- pip3 install sqlmap pymysql # Install sqlmap within the virtual environment
- sqlmap -d "mysql://$DB_USER:$DB_PASSWORD@sdt:3306/$DB_DATABASE" --batch --risk=1 --level=1 --banner --users --dbs --fingerprint
rules:
- if: $CI_COMMIT_BRANCH == "develop"
publish:latest:
stage: publish
image: image: docker:latest
needs:
- test:database
- test:sast
- publish:commit
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA}
- docker tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA} $CI_REGISTRY_IMAGE:$TAG
- docker push $CI_REGISTRY_IMAGE:$TAG
rules:
- if: $CI_COMMIT_BRANCH == "develop"
# --- only main deploys --- #
create:stable:
image: docker:latest
services:
- docker:dind
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:latest
- docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$TAG
- docker push $CI_REGISTRY_IMAGE:$TAG
rules:
- if: $CI_COMMIT_BRANCH == "main"
deploy:k8s:
image: gitlab.ard.de:4567/general/k8s-client
needs:
- create:stable
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
extends: .prep-k8s
script:
- envsubst < k8s/persistentvolume.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
- envsubst < k8s/persistentvolumeclaim.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
- envsubst < k8s/deployment.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
- envsubst < k8s/service.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
release:
needs:
- deploy:k8s
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
script:
- echo "Creating release $VERSION"
release:
tag_name: $VERSION
description: "Release $CI_PROJECT_PATH v$VERSION"
stages:
- lint # Lint MySQL-Files and SAST
- build # Create Container
- scan # Scan Container and push commit
- test # DAST and Acceptance against running container
- publish # publish tages latest or stable
- deploy # To k8s
workflow:
rules:
- if: $CI_COMMIT_REF_NAME == "main" # Main branch
variables:
K8S_CLUSTER: ${K8S_CLUSTER_LIVE}
K8S_CLUSTER_SERVER: ${K8S_CLUSTER_SERVER_LIVE}
K8S_CERTIFICATE_AUTHORITY_DATA: ${K8S_CERTIFICATE_AUTHORITY_DATA_LIVE}
K8S_TOKEN: ${K8S_TOKEN_LIVE}
TAG: stable
TLS_CERT: ${TLS_CERT_LIVE}
TLS_KEY: ${TLS_KEY_LIVE}
ENVIRONMENT_NAME: live
DNS_BACKEND: ${DNS_BACKEND_STAGE}
- if: $CI_COMMIT_REF_NAME == "develop" # Develop branch
variables:
K8S_CLUSTER: ${K8S_CLUSTER_STAGE}
K8S_CLUSTER_SERVER: ${K8S_CLUSTER_SERVER_STAGE}
K8S_CERTIFICATE_AUTHORITY_DATA: ${K8S_CERTIFICATE_AUTHORITY_DATA_STAGE}
K8S_TOKEN: ${K8S_TOKEN_STAGE}
TAG: latest
TLS_CERT: ${TLS_CERT_STAGE}
TLS_KEY: ${TLS_KEY_STAGE}
ENVIRONMENT_NAME: stage
DNS_BACKEND: ${DNS_BACKEND_STAGE}
- if: $CI_COMMIT_REF_NAME != "main" && $CI_COMMIT_REF_NAME != "develop" # Feature / Hotfix branch
variables:
TAG: $CI_COMMIT_SHORT_SHA
.prep-k8s: &prep-k8s # k8s connection settings
before_script:
- kubectl config set-cluster ${K8S_CLUSTER} --server="${K8S_CLUSTER_SERVER}"
- kubectl config set clusters.${K8S_CLUSTER}.certificate-authority-data ${K8S_CERTIFICATE_AUTHORITY_DATA}
- kubectl config set-credentials ${K8S_USER} --token="${K8S_TOKEN}"
- kubectl config set-context k8s --cluster=${K8S_CLUSTER} --user=${K8S_USER} --namespace=${K8S_NAMESPACE}
- kubectl config use-context k8s
# --- ALL branches automatically LINT/SAST ---#
lint:sast:
stage: lint
image: python:3.9-slim
before_script:
- pip install sqlfluff
script:
- sqlfluff lint ./sql/*.sql --dialect postgres
# --- Only Develop Builds and tests --- #
build:
stage: build
image: docker:latest
variables:
FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
BUILD_ARGS:
script:
- echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
- docker build -f ./Containerfile -t ${FULL_IMAGE_NAME} ${BUILD_ARGS} .
- docker save ${FULL_IMAGE_NAME} -o ${OCI_IMAGE_NAME}
artifacts:
paths:
- ${OCI_IMAGE_NAME}
rules:
- if: $CI_COMMIT_BRANCH == "develop"
test:container-scan:
stage: scan
needs:
- job: build
artifacts: true
image:
name: docker.io/aquasec/trivy:latest
entrypoint: [""]
variables:
GIT_STRATEGY: none
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
TRIVY_INSECURE: "true"
script:
- trivy --version
- time trivy image --exit-code 0 --severity HIGH --no-progress --input ${OCI_IMAGE_NAME}
- time trivy image --exit-code 1 --ignore-unfixed --severity CRITICAL --no-progress --input ${OCI_IMAGE_NAME}
rules:
- if: $CI_COMMIT_BRANCH == "develop"
publish:commit:
stage: scan
image: docker:latest
rules:
- if: $CI_COMMIT_BRANCH == "develop"
needs:
- job: build
artifacts: true
- job: test:container-scan
variables:
FULL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
OCI_IMAGE_NAME: ${CI_PROJECT_NAME}-${CI_COMMIT_SHA}.tar
script:
- echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
- docker load -i ${OCI_IMAGE_NAME}
- docker push ${FULL_IMAGE_NAME}
test:database:
stage: test
image: mysql:latest
services:
- name: $CI_REGISTRY_IMAGE:$TAG
alias: db
before_script:
- apt-get update && apt-get install -y postgresql-client # Install PostgreSQL client
script:
- echo "Not Tables to Test"
rules:
- if: $CI_COMMIT_BRANCH == "develop"
test:dast:
stage: test
image: alpine:latest
services:
- name: $CI_REGISTRY_IMAGE:$TAG
alias: db
before_script:
- apk add --no-cache python3 py3-pip python3-dev build-base libffi-dev # Install required packages
script:
- python3 -m venv venv # Create a virtual environment
- source venv/bin/activate # Activate the virtual environment
- pip3 install sqlmap psycopg2 # Install sqlmap within the virtual environment
- sqlmap -d "postgresql://$DB_USER:$DB_PASSWORD@db:5432/$DB_DATABASE" --batch --risk=1 --level=1 --banner --users --dbs --fingerprint
rules:
- if: $CI_COMMIT_BRANCH == "develop"
publish:latest:
stage: publish
image: image: docker:latest
needs:
- test:database
- test:sast
- publish:commit
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA}
- docker tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA} $CI_REGISTRY_IMAGE:$TAG
- docker push $CI_REGISTRY_IMAGE:$TAG
rules:
- if: $CI_COMMIT_BRANCH == "develop"
# --- only main deploys --- #
create:stable:
image: docker:latest
services:
- docker:dind
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:latest
- docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$TAG
- docker push $CI_REGISTRY_IMAGE:$TAG
rules:
- if: $CI_COMMIT_BRANCH == "main"
deploy:k8s:
image: gitlab.ard.de:4567/general/k8s-client
needs:
- create:stable
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
extends: .prep-k8s
script:
- envsubst < k8s/persistentvolume.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
- envsubst < k8s/persistentvolumeclaim.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
- envsubst < k8s/deployment.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
- envsubst < k8s/service.template.yaml | kubectl apply --insecure-skip-tls-verify -f -
release:
needs:
- deploy:k8s
rules:
- if: '$CI_COMMIT_BRANCH == "main"'
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
script:
- echo "Creating release $VERSION"
release:
tag_name: $VERSION
description: "Release $CI_PROJECT_PATH v$VERSION"
#!/bin/bash
set -e
# Wait for MariaDB to be ready
until mariadb -u root -p"$MARIADB_ROOT_PASSWORD" -e ";" ; do
sleep 1
done
# Create Database buzzboard
echo "Creating Database and User ${DB_USER}"
mariadb -u root -p"$MARIADB_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS ${DB_DATABASE};"
mariadb -u root -p"$MARIADB_ROOT_PASSWORD" -e "CREATE USER IF NOT EXISTS ${DB_USER}@'%' IDENTIFIED BY '${DB_PASSWORD}'; "
mariadb -u root -p"$MARIADB_ROOT_PASSWORD" -e "GRANT SELECT, INSERT, UPDATE, DELETE ON ${DB_DATABASE}.* TO ${DB_USER}@'%'; "
\ No newline at end of file
#!/bin/bash
set -e
# Wait for PostgreSQL to be ready
until psql -U "$POSTGRES_USER" -c '\l'; do
sleep 1
done
# Create Database buzzboard
echo "Creating Database and User ${DB_USER}"
psql -U "$POSTGRES_USER" -c "CREATE DATABASE ${DB_DATABASE};"
psql -U "$POSTGRES_USER" -c "CREATE USER ${DB_USER} WITH PASSWORD '${DB_PASSWORD}';"
psql -U "$POSTGRES_USER" -c "GRANT ALL PRIVILEGES ON DATABASE ${DB_DATABASE} TO ${DB_USER};"
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment