From d4b6f785549e02e15029724ef3c4365f6cf64bab Mon Sep 17 00:00:00 2001 From: DeveloperDurp Date: Sun, 9 Apr 2023 11:05:20 -0500 Subject: [PATCH] initial commit --- job-templates.tpl.yml | 28 +++++++++++++++ job-templates.yml | 57 ++++++++++++++++++++++++++++++ pipelines/build-container.yml | 24 +++++++++++++ pipelines/temp | 0 scripts/commit-and-push-changes.sh | 12 +++++++ scripts/docker-build.sh | 5 +++ scripts/docker-login.sh | 5 +++ scripts/install-curl.sh | 5 +++ scripts/install-git.sh | 5 +++ scripts/install-yq.sh | 8 +++++ scripts/render-job-templates.sh | 43 ++++++++++++++++++++++ templates/build-container.tpl.yml | 14 ++++++++ 12 files changed, 206 insertions(+) create mode 100644 job-templates.tpl.yml create mode 100644 job-templates.yml create mode 100644 pipelines/build-container.yml create mode 100644 pipelines/temp create mode 100644 scripts/commit-and-push-changes.sh create mode 100644 scripts/docker-build.sh create mode 100644 scripts/docker-login.sh create mode 100644 scripts/install-curl.sh create mode 100644 scripts/install-git.sh create mode 100644 scripts/install-yq.sh create mode 100644 scripts/render-job-templates.sh create mode 100644 templates/build-container.tpl.yml diff --git a/job-templates.tpl.yml b/job-templates.tpl.yml new file mode 100644 index 0000000..6734825 --- /dev/null +++ b/job-templates.tpl.yml @@ -0,0 +1,28 @@ +# stages are intentionally not defined, as users need to combine the full list +#stages: +# - test +# - render + +workflow: + rules: + # those rules exclude the render commits which are manually triggered in commit-and-push-changes.sh + # being so strict about it prevents "skipped merge request pipelines" which block merging as they're preferred + - if: '$CI_COMMIT_BRANCH && $CI_SCRIPTS_SKIP_PIPELINE != "true"' + +render-job-templates-yml: + stage: render + image: debian:bullseye-slim + script: + - ./scripts/install-curl.sh + - ./scripts/install-git.sh + - ./scripts/install-yq.sh + - ./scripts/render-job-templates.sh + - ./scripts/commit-and-push-changes.sh + rules: + - if: $CI_COMMIT_BRANCH + changes: + - .gitlab-ci.yml + - scripts/** + - templates/** + - job-templates.yml + - job-templates.tpl.yml diff --git a/job-templates.yml b/job-templates.yml new file mode 100644 index 0000000..b4f3542 --- /dev/null +++ b/job-templates.yml @@ -0,0 +1,57 @@ +### WARNING ### +### THIS FILE IS RENDERED! DO NOT EDIT! ANY CHANGE WILL BE REVERTED BY RENDERING PIPELINE +### Edit the template file job-templates.tpl.yml instead! +# stages are intentionally not defined, as users need to combine the full list +#stages: +# - test +# - render +workflow: + rules: + # those rules exclude the render commits which are manually triggered in commit-and-push-changes.sh + # being so strict about it prevents "skipped merge request pipelines" which block merging as they're preferred + - if: '$CI_COMMIT_BRANCH && $CI_SCRIPTS_SKIP_PIPELINE != "true"' +render-job-templates-yml: + stage: render + image: debian:bullseye-slim + script: + # Begin of install-curl.sh + - | + #Install curl + apt update && apt install curl -y + # End of install-curl.sh + + # Begin of install-git.sh + - | + #Install git + apt update && apt install git -y + # End of install-git.sh + + # Begin of install-yq.sh + - "#Install YQ\nexport YQ_VERSION=4.27.2\nexport YQ_ARCH=linux_amd64 \ncurl -fsSL \"https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${YQ_ARCH}\" -o yq \nchmod +x yq\n" + # End of install-yq.sh + + # Begin of render-job-templates.sh + - "# render job-templates.yml from job-templates.tpl.yml\nset -euo pipefail\nfor script in scripts/*.sh; do\n awk -v script_name=\"$(basename \"$script\")\" '\n NR==1 && /^#!/ {printf(\"# Begin of %s\\n\",script_name); next} # strip shebang in first line, print head comment\n /^\\s*$/ {next} # strip any newlines or whitespace\n /^#%%MULTILINE_YAML_START$/ { print \"- |\"; multiline=1; next } # detect start of multiline yaml block, print \"- |\" into output yml\n /^#%%MULTILINE_YAML_END$/ { multiline=0; next } # detect end of multiline yaml block\n multiline==1 {printf(\" %s\\n\",$0)} # print indented script of multiline yaml block\n multiline==0 {printf(\"- !!str %s\\n\",$0)} # not in multiline yaml, just print with \"- !!str\" (explicit yaml tag to avoid any quoting)\n END { printf(\"# End of %s\\n\",script_name) } # print trailing comment\n ' \"$script\" >\"$script.yml\"\ndone\n# start with copy and then edit in place using yq -i\ncat <job-templates.yml\n### WARNING ###\n### THIS FILE IS RENDERED! DO NOT EDIT! ANY CHANGE WILL BE REVERTED BY RENDERING PIPELINE\n### Edit the template file job-templates.tpl.yml instead!\nEOF\ncat job-templates.tpl.yml >> job-templates.yml\n# shellcheck disable=SC2016 # \"unquoted $ warning\"\n./yq -i '(.[] | select(keys | .[] | select(tag == \"!!str\") | test(\"^(before_|after_|)script$\")) | .\"*script\") ref $scripts\n | ($scripts | .. | select(. == \"./scripts/*.sh\")) |= (load(. + \".yml\") | .[] style=\"\")\n | ($scripts | .[] | select(type == \"!reference\")) |= ([.] | . style=\"flow\")\n | ($scripts | select(type == \"!!seq\")) |= flatten(1)\n | explode .\n ' job-templates.yml\nfor template in templates/*.tpl.yml\n do \n export pipeline=$(basename $template | sed \"s/.tpl//\")\n cat $template > pipelines/$pipeline \n ./yq -i '(.[] | select(keys | .[] | select(tag == \"!!str\") | test(\"^(before_|after_|)script$\")) | .\"*script\") ref $scripts\n | ($scripts | .. | select(. == \"./scripts/*.sh\")) |= (load(. + \".yml\") | .[] style=\"\")\n | ($scripts | .[] | select(type == \"!reference\")) |= ([.] | . style=\"flow\")\n | ($scripts | select(type == \"!!seq\")) |= flatten(1)\n | explode .\n ' pipelines/$pipeline\ndone\n" + # End of render-job-templates.sh + + # Begin of commit-and-push-changes.sh + - | + # commit and push changes in job-templates.yml (if any) + git config user.name "DeveloperDurpBot" + git config user.email "DeveloperDurp@durp.info" + git add -A + git commit --untracked-files=no -m "ci: render" -m "Rendered by $CI_PIPELINE_URL" || exit 0 + echo "Pushing to branch $CI_COMMIT_BRANCH" + # just using -o ci.skip would create a "skipped" pipeline in the list of pipelines, preventing a merge! + # passing down CI_SCRIPTS_SKIP_PIPELINE and using this in workflow:rules is better, as it does not create a pipeline at all! + git push -o ci.variable="CI_SCRIPTS_SKIP_PIPELINE=true" "https://oauth2:${GITLAB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "HEAD:$CI_COMMIT_BRANCH" + curl --silent --fail --request POST --form token="$CI_JOB_TOKEN" --form ref="$CI_COMMIT_BRANCH" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline" >/dev/null + # End of commit-and-push-changes.sh + rules: + - if: $CI_COMMIT_BRANCH + changes: + - .gitlab-ci.yml + - scripts/** + - templates/** + - job-templates.yml + - job-templates.tpl.yml diff --git a/pipelines/build-container.yml b/pipelines/build-container.yml new file mode 100644 index 0000000..514a747 --- /dev/null +++ b/pipelines/build-container.yml @@ -0,0 +1,24 @@ +build-container: + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: tcp://docker:2375 + services: + - docker:19.03.12-dind + stage: package + script: + # Begin of docker-login.sh + - | + #login to docker + docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY/$CI_PROJECT_PATH + # End of docker-login.sh + + # Begin of docker-build.sh + - | + #Build Docker Container + docker build -t $CI_REGISTRY/$CI_PROJECT_PATH:$DOCKERTAG -t $CI_REGISTRY/$CI_PROJECT_PATH:latest . + docker push "$CI_REGISTRY/$CI_PROJECT_PATH:latest" + # End of docker-build.sh + only: + - main diff --git a/pipelines/temp b/pipelines/temp new file mode 100644 index 0000000..e69de29 diff --git a/scripts/commit-and-push-changes.sh b/scripts/commit-and-push-changes.sh new file mode 100644 index 0000000..5a252d2 --- /dev/null +++ b/scripts/commit-and-push-changes.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +# commit and push changes in job-templates.yml (if any) +git config user.name "DeveloperDurpBot" +git config user.email "DeveloperDurp@durp.info" +git add -A +git commit --untracked-files=no -m "ci: render" -m "Rendered by $CI_PIPELINE_URL" || exit 0 +echo "Pushing to branch $CI_COMMIT_BRANCH" +# just using -o ci.skip would create a "skipped" pipeline in the list of pipelines, preventing a merge! +# passing down CI_SCRIPTS_SKIP_PIPELINE and using this in workflow:rules is better, as it does not create a pipeline at all! +git push -o ci.variable="CI_SCRIPTS_SKIP_PIPELINE=true" "https://oauth2:${GITLAB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "HEAD:$CI_COMMIT_BRANCH" +curl --silent --fail --request POST --form token="$CI_JOB_TOKEN" --form ref="$CI_COMMIT_BRANCH" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline" >/dev/null diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh new file mode 100644 index 0000000..97d16f9 --- /dev/null +++ b/scripts/docker-build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +#Build Docker Container +docker build -t $CI_REGISTRY/$CI_PROJECT_PATH:$DOCKERTAG -t $CI_REGISTRY/$CI_PROJECT_PATH:latest . +docker push "$CI_REGISTRY/$CI_PROJECT_PATH:latest" \ No newline at end of file diff --git a/scripts/docker-login.sh b/scripts/docker-login.sh new file mode 100644 index 0000000..d5bf236 --- /dev/null +++ b/scripts/docker-login.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +#login to docker + +docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY/$CI_PROJECT_PATH diff --git a/scripts/install-curl.sh b/scripts/install-curl.sh new file mode 100644 index 0000000..76d3f2f --- /dev/null +++ b/scripts/install-curl.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +#Install curl + +apt update && apt install curl -y \ No newline at end of file diff --git a/scripts/install-git.sh b/scripts/install-git.sh new file mode 100644 index 0000000..1c78b04 --- /dev/null +++ b/scripts/install-git.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +#Install git + +apt update && apt install git -y \ No newline at end of file diff --git a/scripts/install-yq.sh b/scripts/install-yq.sh new file mode 100644 index 0000000..577640e --- /dev/null +++ b/scripts/install-yq.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +#Install YQ + +export YQ_VERSION=4.27.2 +export YQ_ARCH=linux_amd64 +curl -fsSL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${YQ_ARCH}" -o yq +chmod +x yq \ No newline at end of file diff --git a/scripts/render-job-templates.sh b/scripts/render-job-templates.sh new file mode 100644 index 0000000..bce19d2 --- /dev/null +++ b/scripts/render-job-templates.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +#%%MULTILINE_YAML_START +# render job-templates.yml from job-templates.tpl.yml +set -euo pipefail +for script in scripts/*.sh; do + awk -v script_name="$(basename "$script")" ' + NR==1 && /^#!/ {printf("# Begin of %s\n",script_name); next} # strip shebang in first line, print head comment + /^\s*$/ {next} # strip any newlines or whitespace + /^#%%MULTILINE_YAML_START$/ { print "- |"; multiline=1; next } # detect start of multiline yaml block, print "- |" into output yml + /^#%%MULTILINE_YAML_END$/ { multiline=0; next } # detect end of multiline yaml block + multiline==1 {printf(" %s\n",$0)} # print indented script of multiline yaml block + multiline==0 {printf("- !!str %s\n",$0)} # not in multiline yaml, just print with "- !!str" (explicit yaml tag to avoid any quoting) + END { printf("# End of %s\n",script_name) } # print trailing comment + ' "$script" >"$script.yml" +done +# start with copy and then edit in place using yq -i +cat <job-templates.yml +### WARNING ### +### THIS FILE IS RENDERED! DO NOT EDIT! ANY CHANGE WILL BE REVERTED BY RENDERING PIPELINE +### Edit the template file job-templates.tpl.yml instead! + +EOF +cat job-templates.tpl.yml >> job-templates.yml + +# shellcheck disable=SC2016 # "unquoted $ warning" +./yq -i '(.[] | select(keys | .[] | select(tag == "!!str") | test("^(before_|after_|)script$")) | ."*script") ref $scripts + | ($scripts | .. | select(. == "./scripts/*.sh")) |= (load(. + ".yml") | .[] style="") + | ($scripts | .[] | select(type == "!reference")) |= ([.] | . style="flow") + | ($scripts | select(type == "!!seq")) |= flatten(1) + | explode . + ' job-templates.yml + +for template in templates/*.tpl.yml + do + export pipeline=$(basename $template | sed "s/.tpl//") + cat $template > pipelines/$pipeline + ./yq -i '(.[] | select(keys | .[] | select(tag == "!!str") | test("^(before_|after_|)script$")) | ."*script") ref $scripts + | ($scripts | .. | select(. == "./scripts/*.sh")) |= (load(. + ".yml") | .[] style="") + | ($scripts | .[] | select(type == "!reference")) |= ([.] | . style="flow") + | ($scripts | select(type == "!!seq")) |= flatten(1) + | explode . + ' pipelines/$pipeline +done diff --git a/templates/build-container.tpl.yml b/templates/build-container.tpl.yml new file mode 100644 index 0000000..ffad142 --- /dev/null +++ b/templates/build-container.tpl.yml @@ -0,0 +1,14 @@ +build-container: + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: tcp://docker:2375 + services: + - docker:19.03.12-dind + stage: package + script: + - ./scripts/docker-login.sh + - ./scripts/docker-build.sh + only: + - main \ No newline at end of file