diff --git a/.gitignore b/.gitignore index 2148f30ba983c315d068e282888eb3ad968617b3..c8306e1484f23f0ee1366455c951cf3acb6c891c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ **/secrets/* -.idea \ No newline at end of file +.idea +/data/ +/tmp/ diff --git a/README-distros.md b/README-distros.md new file mode 100644 index 0000000000000000000000000000000000000000..e01198eb7f06949b2d3ae0b224012c8f0f361d5a --- /dev/null +++ b/README-distros.md @@ -0,0 +1,83 @@ + +Minikube + +### Fedora 39 !!! +install minikube + +https://kubernetes.io/ru/docs/tasks/tools/install-minikube/ + +then +https://docs.fedoraproject.org/en-US/quick-docs/using-kubernetes/ + +``` +sudo dnf install kubernetes-client kubernetes-node kubernetes-kubeadm +sudo dnf install cri-o containernetworking-plugins +``` + +``` +sudo systemctl enable --now crio +sudo systemctl enable --now kubelet +``` + +``` +minikube start +``` + +port forward +``` +firewall-cmd --zone=external --add-forward-port=port=8443:proto=tcp:toaddr=192.168.49.2 +``` + +enp1s0: +# kube API 8443 + +sudo iptables -A PREROUTING -t nat -i enp1s0 -p tcp --dport 8443 -j DNAT --to 192.168.49.2:8443 +sudo iptables -A FORWARD -p tcp -d 192.168.49.2 --dport 8443 -j ACCEPT + +## startup +https://operavps.com/docs/run-command-after-boot-in-linux/ + +``` +nmap -n -PN -sT -sU -p- localhost +``` +При Ñтарте миникуба в федоре +``` +minikube start --embed-certs \ +--apiserver-name k8s.codemonsters.team \ +--driver=podman \ +--addons=['helm-tiller','ingress'] +--container-runtime=containerd +``` +Ðе поднимаетÑÑ Ñетевой Ð¸Ð½Ñ‚ÐµÑ€Ñ„ÐµÐ¹Ñ Ð¸ ÐºÐ¾Ð½Ñ„Ð¸Ð³ÑƒÑ€Ð°Ñ†Ð¸Ñ Ð½Ðµ работает + +install docker +https://docs.docker.com/engine/install/fedora/ +post-install +https://docs.docker.com/engine/install/linux-postinstall/ +миникуб на docker-e поднимаетÑÑ Ð±Ñ‹Ñтрее по ощущениÑм +нужно бенчмарк Ñделать + +## minikube on podman startup time +Benchmark start minikube on localhost +```bash +strace -o trace -c -Ttt /var/opt/developer-sandbox/init-developer-sandbox-podman.sh +``` +result: + +| time | seconds | usecs/call | calls | errors syscall | +| ------ | ---------- | ----------- | ------- | ---------------- | +| 100.00 | 1.609212 | 7216 | 223 | 20 total | +[details-trace-podman](research/minikube-startup-trace-podman.txt) + +## minikube on docker startup time +```bash +strace -o trace -c -Ttt /var/opt/developer-sandbox/init-developer-sandbox-docker.sh + +``` +| time | seconds | usecs/call | calls | errors syscall | +| ------ | ---------- | ----------- | ------- | ---------------- | +| 100.00 | 0.677493 | 3051 | 222 | 20 total | +[details-trace-docker](research/minikube-startup-trace-docker.txt) + +## install tools +https://kubernetes.io/docs/tasks/tools/ diff --git a/README.md b/README.md index f873e0ccd8a553861959e296601ad93ab6109738..a951783e4119b203690b64ecdd29e2db8655313a 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ <p>Ð’ образовательных целÑÑ… Ñтроим пеÑочницу на одном Ñерваке за 70К: - Docker, Docker swarm - kubernetes (minikube) -- gitlab Ñ Ð¸Ð½Ñ‚ÐµÐ³Ñ€Ð°Ñ†Ð¸ÐµÐ¹ по OpenId Ñ keycloack +- gitlab Ñ Ð¸Ð½Ñ‚ÐµÐ³Ñ€Ð°Ñ†Ð¸ÐµÐ¹ по OpenId Ñ keycloak - gitlab runner -- keycloack +- keycloak - openproject(?) Также цель иÑÑледовать такие инженерные практики как Infrastructure As a Code, СI, CD. </p> @@ -38,7 +38,7 @@ graph TD; A[Internet Gateway:443]-->B[Server:4443]; B-->C[Nginx:4443 apply ssl]; C-->D[Gitlab in Docker:8000]; - C-->E[keycloack in Docker:8888]; + C-->E[keycloak in Docker:8888]; C-->F[minikube for stateless services:80]; C-->G[plantuml in Docker:8001]; ``` @@ -46,10 +46,26 @@ graph TD; # TODO - [x] перенеÑти certbot в докер - - [ ] Ñоздать Ñкрипт накатки конфига на Ñервер - - [ ] протеÑтировать накатку конфигурации на Ñервер - - [ ] наÑтроить бÑкап + - [ ] Ñоздать docker-compose накатки Ñтаи на Ñервер + - [x] certbot + - [x] keycloak + - [x] gitlab + - [x] gitlab-runner + - [x] nginx + - [x] redirect to k8s API [configuring-tcp-or-udp-load-balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/#configuring-tcp-or-udp-load-balancing) + - [ ] инÑÑ‚Ñ€ÑƒÐºÑ†Ð¸Ñ Ð¿Ð¾ Ñозданию Ñекретов + - [ ] ÐºÐ¾Ð½Ñ„Ð¸Ð³ÑƒÑ€Ð°Ñ†Ð¸Ñ Ñ€ÐµÐ³ÑƒÐ»Ñрного Ñтарта certbot renew + - [x] наÑтроить Ñеть sandbox-a + - [x] перенаправить трафик + - [x] региÑÑ‚Ñ€Ð°Ñ†Ð¸Ñ gitlab-runner + - [ ] Ñоздание раннеров по API + Ñкрипт региÑтрации + - [x] протеÑтировать накатку конфигурации на Ñервер + - [x] наÑтроить бÑкап + - [ ] наÑтроить бÑкап по Ñети + - [ ] воÑÑтановление из бÑкапа - [ ] теÑтировать воÑÑтановление + - [ ] запуÑк Ð´ÐµÐ¿Ð»Ð¾Ñ Ñайта + - [ ] мониторинг # Research - [ ] https://c4model.com/ @@ -57,8 +73,26 @@ graph TD; # Main Pipeline - [ ] запуÑк certbot по вÑем Ñертификатам - - [ ] запуÑк keycloack + - [ ] запуÑк keycloak - [ ] запуÑк gitlab - [ ] запуÑк gitlab-runner и региÑÑ‚Ñ€Ð°Ñ†Ð¸Ñ Ð² гитлаб автоматом - [ ] запуÑк minikube - [ ] запуÑк nginx + +## Backup +run as a root +```zsh +55 23 * * * /var/opt/developer-sandbox/backup/backup.sh +``` + +run as a user +## Certbot renew + +```zsh +55 23 * * * /var/opt/developer-sandbox/certbot/renew-certs.sh +``` + +## Feature + - [ ] https://k0sproject.io/ + - [ ] https://docs.k0sproject.io/v1.28.6+k0s.0/ + - [ ] https://kind.sigs.k8s.io/docs/user/quick-start/ \ No newline at end of file diff --git a/backup/backup.sh b/backup/backup.sh new file mode 100755 index 0000000000000000000000000000000000000000..8a4ecfcc2f43b7d9c51d7f6b71215ebf572a5ec2 --- /dev/null +++ b/backup/backup.sh @@ -0,0 +1,15 @@ +archive_name=$(date +%d%m%Y%T) +mkdir /var/opt/backup/$archive_name/ + +rsync -azvP /var/opt/developer-sandbox/data/gitlab-data /var/opt/backup/$archive_name/ + +#runners +rsync -azvP /var/lib/docker/volumes/developer-sandbox_gitlab-runner-config-0 /var/opt/backup/$archive_name/ +rsync -azvP /var/lib/docker/volumes/developer-sandbox_gitlab-runner-config-1 /var/opt/backup/$archive_name/ +#tar -czvf gitlab-data.tar.gz gitlab-backup + +#keycloak +rsync -azvP /var/lib/docker/volumes/developer-sandbox_keycloak-data /var/opt/backup/$archive_name/ + +#certbot +rsync -azvP /var/opt/developer-sandbox/data/certbot /var/opt/backup/$archive_name/ diff --git a/backup/certbot.sh b/backup/certbot.sh new file mode 100755 index 0000000000000000000000000000000000000000..c132a75c2f40f2c237ec8c575bcd0a5bc562987b --- /dev/null +++ b/backup/certbot.sh @@ -0,0 +1 @@ +rsync -azvP /var/opt/developer-sandbox/data/certbot /var/opt/backup/ diff --git a/backup/keycloak.sh b/backup/keycloak.sh new file mode 100755 index 0000000000000000000000000000000000000000..a9ee797b87a1846536344228b89489eba12accc9 --- /dev/null +++ b/backup/keycloak.sh @@ -0,0 +1,2 @@ +rsync -azvP /var/lib/docker/volumes/developer-sandbox_keycloak-data /var/opt/backup/ +#tarmk -czvf keycloak-data.tar.gz keycloak-backup diff --git a/certbot/README.md b/certbot/README.md index f97f145acc591d47c838b1b698d75892a1820764..9adbc05aa6f6d8fbfcc8c1a742e87b38a1ae0866 100644 --- a/certbot/README.md +++ b/certbot/README.md @@ -30,6 +30,7 @@ docker run -it --rm --name certbot-dns-digitalocean \ --email maxim@codemonsters.team \ --agree-tos \ --dns-digitalocean \ - --domain git.codemonsters.team \ + -d codemonsters.team \ + -d *.codemonsters.team \ --dns-digitalocean-credentials /var/lib/letsencrypt/digitalocean.ini ```` diff --git a/certbot/docker-compose-certbot-renew.yml b/certbot/docker-compose-certbot-renew.yml new file mode 100644 index 0000000000000000000000000000000000000000..3de0502938dd9d748d0323aca5f45b239dd6a4dd --- /dev/null +++ b/certbot/docker-compose-certbot-renew.yml @@ -0,0 +1,30 @@ +version: "3.8" +services: + certbot-renew: + image: certbot/dns-digitalocean + container_name: certbot-dns-digitalocean-renew + restart: no + hostname: certbot-dns-digitalocean-renew + volumes: + - /var/opt/developer-sandbox/data/certbot/etc:/etc/letsencrypt + - /var/opt/developer-sandbox/data/certbot/lib:/var/lib/letsencrypt + - /var/opt/developer-sandbox/data/certbot/log:/var/log/letsencrypt + secrets: + - dns-digitalocean-credentials + command: "certonly \ + --noninteractive \ + --email maxim@codemonsters.team + --agree-tos \ + -d codemonsters.team \ + -d *.codemonsters.team \ + --dns-digitalocean \ + --dns-digitalocean-credentials /run/secrets/dns-digitalocean-credentials" + networks: + - mechanicus-swarm + +secrets: + dns-digitalocean-credentials: + file: /var/opt/developer-sandbox/certbot/secrets/digitalocean.ini + +networks: + mechanicus-swarm: \ No newline at end of file diff --git a/certbot/docker-compose.yml b/certbot/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..82e6f04f97fbfb5a24b983d714aeb0cb422edafd --- /dev/null +++ b/certbot/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3.8" +services: + certbot: + image: certbot/dns-digitalocean + container_name: certbot-dns-digitalocean + restart: always + hostname: certbot-dns-digitalocean + volumes: + - /var/opt/developer-sandbox/data/certbot/etc:/etc/letsencrypt + - /var/opt/developer-sandbox/data/certbot/lib:/var/lib/letsencrypt + - /var/opt/developer-sandbox/data/certbot/log:/var/log/letsencrypt + secrets: + - dns-digitalocean-credentials + command: "certonly \ + --noninteractive \ + --email maxim@codemonsters.team + --agree-tos \ + -d codemonsters.team \ + -d *.codemonsters.team \ + --dns-digitalocean \ + --dns-digitalocean-credentials /run/secrets/dns-digitalocean-credentials" + networks: + - intranet +secrets: + dns-digitalocean-credentials: + file: ./secrets/digitalocean.ini + +networks: + intranet: \ No newline at end of file diff --git a/certbot/renew-certs.sh b/certbot/renew-certs.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b50d5d937e789739ae87ea35b7ef85aa1c99b0f --- /dev/null +++ b/certbot/renew-certs.sh @@ -0,0 +1,2 @@ +docker compose -f /var/opt/developer-sandbox/certbot/docker-compose-certbot-renew.yml up +docker compose -f /var/opt/developer-sandbox/certbot/docker-compose-certbot-renew.yml down \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..0301a7a2fda93a1f6f3abb9c9282df07ea2b939d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,176 @@ +version: "3.8" +services: + certbot: + image: certbot/dns-digitalocean + container_name: certbot-dns-digitalocean + restart: no + hostname: certbot-dns-digitalocean + volumes: + - /var/opt/developer-sandbox/data/certbot/etc:/etc/letsencrypt + - /var/opt/developer-sandbox/data/certbot/lib:/var/lib/letsencrypt + - /var/opt/developer-sandbox/data/certbot/log:/var/log/letsencrypt + secrets: + - dns-digitalocean-credentials + command: "certonly \ + --noninteractive \ + --email maxim@codemonsters.team + --agree-tos \ + -d codemonsters.team \ + -d *.codemonsters.team \ + --dns-digitalocean \ + --dns-digitalocean-credentials /run/secrets/dns-digitalocean-credentials" + networks: + - mechanicus-swarm + + keycloak: + image: quay.io/keycloak/keycloak:22.0 + container_name: keycloak-dev + restart: always + command: + - "start-dev" + secrets: + - keycloak_admin_name + - keycloak_admin_password + environment: + KEYCLOAK_ADMIN: /run/secrets/keycloak_admin_name + KEYCLOAK_ADMIN_PASSWORD: /run/secrets/keycloak_admin_password + KC_HOSTNAME_URL: "https://key.codemonsters.team" + KC_HOSTNAME_STRICT_HTTPS: false + KC_PROXY: edge + ports: + - "8888:8080" + volumes: + - type: volume + source: keycloak-data + target: /opt/keycloak/data + read_only: false + networks: + - mechanicus-swarm + depends_on: [ certbot ] + + gitlab: + image: gitlab/gitlab-ce:16.8.1-ce.0 + container_name: gitlab + restart: always + hostname: gitlab + ports: + - "3333:22" + - "8000:80" + - "2443:443" + volumes: + - /var/opt/developer-sandbox/data/gitlab-data/data:/var/opt/gitlab + - /var/opt/developer-sandbox/data/gitlab-data/logs:/var/log/gitlab + - /var/opt/developer-sandbox/data/gitlab-data/config:/etc/gitlab + shm_size: '256m' + environment: + GITLAB_OMNIBUS_CONFIG: "from_file('/omnibus_config.rb')" + configs: + - source: gitlab + target: /omnibus_config.rb + secrets: + - gitlab_root_password + - smtp_user + - smtp_password + networks: + - mechanicus-swarm + depends_on: [ certbot, keycloak ] + + plantuml: + image: 'plantuml/plantuml-server:tomcat' + container_name: plantuml + restart: always + ports: + - "8001:8080" + networks: + - mechanicus-swarm + + gitlab-runner-0: + restart: always + image: gitlab/gitlab-runner:latest + hostname: gitlab-runner-0 + volumes: + - type: volume + source: gitlab-runner-config-0 + target: /etc/gitlab-runner + read_only: false + - /var/run/docker.sock:/var/run/docker.sock + networks: + - mechanicus-swarm + depends_on: [ gitlab ] + + gitlab-runner-1: + restart: always + image: gitlab/gitlab-runner:latest + hostname: gitlab-runner-1 + volumes: + - type: volume + source: gitlab-runner-config-1 + target: /etc/gitlab-runner + read_only: false + - /var/run/docker.sock:/var/run/docker.sock + networks: + - mechanicus-swarm + depends_on: [ gitlab ] + + nginx: + image: nginx:stable-alpine3.17 + container_name: nginx + #restart: always -- first you need to start minikube, then start nginx + #look at the minikube launch script ../minikube/minikube-start.sh + hostname: nginx + secrets: + - monsters-certificate-key + - monsters-certificate-fullchain + volumes: + - type: bind + source: /var/opt/developer-sandbox/nginx/nginx + target: /etc/nginx + read_only: true + - type: volume + source: nginx-log + target: /var/logs/nginx + read_only: false + ports: + - "4443:4443" + - "8443:8443" + networks: + - k8s.codemonsters.team + - mechanicus-swarm + depends_on: [ certbot, gitlab, keycloak, plantuml ] + +configs: + gitlab: + file: gitlab/gitlab.rb + +secrets: + dns-digitalocean-credentials: + file: /var/opt/developer-sandbox/certbot/secrets/digitalocean.ini + keycloak_admin_name: + file: /var/opt/developer-sandbox/keycloak/secrets/admin_name.txt + keycloak_admin_password: + file: /var/opt/developer-sandbox/keycloak/secrets/admin_password.txt + gitlab_root_password: + file: /var/opt/developer-sandbox/gitlab/secrets/root_password.txt + smtp_user: + file: /var/opt/developer-sandbox/gitlab/secrets/smtp_user.txt + smtp_password: + file: /var/opt/developer-sandbox/gitlab/secrets/smtp_password.txt + gitlab_register_token: + file: /var/opt/developer-sandbox/gitlab-runner/secrets/gitlab_register_token.txt + ci_server_url: + file: /var/opt/developer-sandbox/gitlab-runner/secrets/ci_server_url.txt + monsters-certificate-key: + file: /var/opt/developer-sandbox/data/certbot/etc/live/codemonsters.team/privkey.pem + monsters-certificate-fullchain: + file: /var/opt/developer-sandbox/data/certbot/etc/live/codemonsters.team/fullchain.pem + +volumes: + keycloak-data: + gitlab-runner-config-0: + gitlab-runner-config-1: + nginx-log: + +networks: + mechanicus-swarm: + k8s.codemonsters.team: + external: true diff --git a/gitlab-runner/README.md b/gitlab-runner/README.md index a623699da8e4f9dc77b160019ee5eccc9f34d8b4..2ca2f29b8574cfabbb9c882a57b3e9473ac958d1 100644 --- a/gitlab-runner/README.md +++ b/gitlab-runner/README.md @@ -50,3 +50,17 @@ gitlab-runner register \ ```` Ðе Ñмог найти путь региÑтрации через compose https://gist.github.com/benoitpetit/cbe19cdd369ec8c1e0defd245d91751f + +From gitlab api: +```` +gitlab-runner +register --url https://git.codemonsters.team +--token <token-1> +```` + +```` +gitlab-runner +register +--url https://git.codemonsters.team +--token <token-2> +```` diff --git a/gitlab-runner/docker-compose-register-runner.yml b/gitlab-runner/docker-compose-register-runner.yml new file mode 100644 index 0000000000000000000000000000000000000000..d1165bd526c4b8f34848ec3ea1f65020961187c6 --- /dev/null +++ b/gitlab-runner/docker-compose-register-runner.yml @@ -0,0 +1,52 @@ +version: "3.8" +services: + gitlab-runner-0: + restart: always + image: gitlab/gitlab-runner:latest + hostname: gitlab-runner-0 + volumes: + - type: volume + source: gitlab-runner-config-0 + target: /etc/gitlab-runner + read_only: false + - /var/run/docker.sock:/var/run/docker.sock + networks: + - developer-sandbox_mechanicus-swarm + +# gitlab-runner-0-register: +# restart: 'no' +# image: gitlab/gitlab-runner:latest +# environment: +# CI_SERVER_URL: "https://git.codemonsters.team" +# command: +# - register +# - --token "$(cat /var/run/secrets/gitlab_register_token)" +# - --non-interactive +# - --locked=false +# - --name="gitlab-runner-0" +# - --description "Docker Gitlab Runner Zero-0" +# - --executor "docker" +# - --docker-image "docker:24.0.6" +# - --docker-volumes=/var/run/docker.sock:/var/run/docker.sock +# secrets: +# - gitlab_register_token +# volumes: +# - type: volume +# source: gitlab-runner-config-0 +# target: /etc/gitlab-runner +# read_only: false +# - /var/run/docker.sock:/var/run/docker.sock +# networks: +# - developer-sandbox_mechanicus-swarm +# depends_on: [ gitlab-runner-0 ] + +secrets: + gitlab_register_token: + file: /var/opt/developer-sandbox/gitlab-runner/secrets/zero-runner-0-token.txt + +volumes: + gitlab-runner-config-0: + +networks: + developer-sandbox_mechanicus-swarm: + external: true diff --git a/gitlab-runner/docker-compose.yml b/gitlab-runner/docker-compose.yml index bf80ce7f098379d7f5845df5e02660c290de0e58..af6d1288c5904b8d2038c4b66582f3cd39f1b56d 100644 --- a/gitlab-runner/docker-compose.yml +++ b/gitlab-runner/docker-compose.yml @@ -1,26 +1,14 @@ version: "3.9" services: - gitlab-runner-0: - restart: always - image: gitlab/gitlab-runner:latest - hostname: gitlab-runner-0 - volumes: - - type: volume - source: gitlab-runner-config-0 - target: /etc/gitlab-runner - read_only: false - - /var/run/docker.sock:/var/run/docker.sock - networks: - - intranet + 16: + 6: + 6-ce: + 0: + volumes: gitlab-runner-config-0: -secrets: - gitlab_register_token: - file: secrets/gitlab_register_token.txt - ci_server_url: - file: secrets/ci_server_url.txt - networks: - intranet: + developer-sandbox_mechanicus-swarm: + external: true diff --git a/gitlab-runner/create-runner.sh b/gitlab-runner/register-runner.sh similarity index 54% rename from gitlab-runner/create-runner.sh rename to gitlab-runner/register-runner.sh index 10d8613b77074604dd777fc7d32e974823adbed6..0692c78fab3ce2fe57d1838deb9c91b267599fcc 100755 --- a/gitlab-runner/create-runner.sh +++ b/gitlab-runner/register-runner.sh @@ -1,13 +1,13 @@ #!/bin/bash -doccker compose up -d docker run --rm -it \ --v gitlab-runner-config-"$3":/etc/gitlab-runner \ +-v "developer-sandbox_gitlab-runner-config-$2":/etc/gitlab-runner \ gitlab/gitlab-runner:latest \ register -n \ ---url "$1" \ ---registration-token "$2" \ +--url "$0" \ +--token "$1" \ --executor docker \ ---description "Docker Gitlab Runner Zero:$3" \ +--description "Docker Gitlab Runner Zero $2" \ --docker-image "docker:24.0.6" \ --docker-volumes /var/run/docker.sock:/var/run/docker.sock + diff --git a/gitlab/docker-compose.yml b/gitlab/docker-compose.yml index 10f2eeaa1b08c7ce025cd60ddbdb65c6cd8c9fd1..4a70987ac10fcc88ed4d1d7f3b87ad52f87d38ea 100644 --- a/gitlab/docker-compose.yml +++ b/gitlab/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.9" services: gitlab: - image: gitlab/gitlab-ce:latest + image: gitlab/gitlab-ce:16.4.1-ce.0 container_name: gitlab restart: always hostname: gitlab @@ -10,9 +10,9 @@ services: - "8000:80" - "2443:443" volumes: - - /opt/devops/gitlab-data/data:/var/opt/gitlab - - /opt/devops/gitlab-data/logs:/var/log/gitlab - - /opt/devops/gitlab-data/config:/etc/gitlab + - /var/opt/developer-sandbox/data/gitlab-data/data:/var/opt/gitlab + - /var/opt/developer-sandbox/data/gitlab-data/logs:/var/log/gitlab + - /var/opt/developer-sandbox/data/gitlab-data/config:/etc/gitlab shm_size: '256m' environment: GITLAB_OMNIBUS_CONFIG: "from_file('/omnibus_config.rb')" @@ -24,23 +24,19 @@ services: - smtp_user - smtp_password networks: - - intranet - plantuml: - image: 'plantuml/plantuml-server:tomcat' - container_name: plantuml - ports: - - "8001:8080" + - mechanicus-swarm configs: gitlab: - file: gitlab.rb + file: gitlab/gitlab.rb + secrets: gitlab_root_password: - file: /opt/devops/gitlab/secrets/root_password.txt + file: /var/opt/developer-sandbox/gitlab/secrets/root_password.txt smtp_user: - file: /opt/devops/gitlab/secrets/smtp_user.txt + file: /var/opt/developer-sandbox/gitlab/secrets/smtp_user.txt smtp_password: - file: /opt/devops/gitlab/secrets/smtp_password.txt + file: /var/opt/developer-sandbox/gitlab/secrets/smtp_password.txt networks: - intranet: \ No newline at end of file + mechanicus-swarm: diff --git a/gitlab/gitlab.rb b/gitlab/gitlab.rb index 7e2667ac58012ddbbb7e7dd0823e3a81e28db9d3..3c8d70fb0e2d95b4192935a4f888df455e2fd3f6 100644 --- a/gitlab/gitlab.rb +++ b/gitlab/gitlab.rb @@ -5,7 +5,8 @@ letsencrypt['enable'] = false nginx['listen_port'] = 80 nginx['listen_https'] = false #plantuml -nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n rewrite ^/-/plantuml/(.*) /$1 break;\n proxy_cache off; \n proxy_pass http://192.168.50.98:8001/; \n}\n" +nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n rewrite ^/-/plantuml/(.*) /$1 break;\n proxy_cache off; \n proxy_pass https://plantuml.codemonsters.team/; \n}\n" +nginx['client_max_body_size'] = '512m' gitlab_rails['initial_root_password'] = File.read('/run/secrets/gitlab_root_password').gsub("\n", "") diff --git a/init-developer-sandbox.sh b/init-developer-sandbox.sh new file mode 100755 index 0000000000000000000000000000000000000000..a94af38803d63c876bf6709bca6b404c9167b42d --- /dev/null +++ b/init-developer-sandbox.sh @@ -0,0 +1,33 @@ +#!/bin/bash +git clone https://git.codemonsters.team/devops/developer-sandbox.git /var/opt/developer-sandbox +mkdir -p /var/opt/developer-sandbox/data/ +mkdir -p /var/opt/developer-sandbox/data/certbot/etc +mkdir -p /var/opt/developer-sandbox/data/certbot/lib +mkdir -p /var/opt/developer-sandbox/data/certbot/log +# adding k8s.mechanicus hosts +sudo bash zero-server/modify-hosts.sh + +# create minikube +/var/opt/developer-sandbox/minikube/minikube-config.sh +/var/opt/developer-sandbox/minikube/minikube-init-start.sh +/var/opt/developer-sandbox/minikube/minikube-enable-addons.sh +# + +## TODO: instructions for adding secrets for compose! +mkdir -p /var/opt/developer-sandbox/certbot/secrets/ +#add secrets +mkdir -p /var/opt/developer-sandbox/gitlab/secrets/ +#add secrets +mkdir -p /var/opt/developer-sandbox/keycloak/secrets/ +#add secrets +mkdir -p /var/opt/developer-sandbox/gitlab-runner/secrets/ +#add secrets +## +# I can run it via cron once a day +# first run certbot to create certs +docker compose -f /var/opt/developer-sandbox/docker-compose-start-certbot.yml up -d +# then run swarm with secrets from certs +docker compose -f /var/opt/developer-sandbox/docker-compose.yml up -d + +#show listened ports on ip +nmap mechanicus diff --git a/keycloack/README.md b/keycloak/README.md similarity index 100% rename from keycloack/README.md rename to keycloak/README.md diff --git a/keycloack/docker-compose.yml b/keycloak/docker-compose.yml similarity index 58% rename from keycloack/docker-compose.yml rename to keycloak/docker-compose.yml index bbde672be89f36d155413355c5d7ddf8f07cb6cc..b41683b1608baf0c2f8ddd5b8ad70d9791e21d3d 100644 --- a/keycloack/docker-compose.yml +++ b/keycloak/docker-compose.yml @@ -7,11 +7,11 @@ services: command: - "start-dev" secrets: - - keycloack_admin_name - - keycloack_admin_password + - _admin_name + - keycloak_admin_password environment: - KEYCLOAK_ADMIN: /run/secrets/keycloack_admin_name - KEYCLOAK_ADMIN_PASSWORD: /run/secrets/keycloack_admin_password + KEYCLOAK_ADMIN: /run/secrets/keycloak_admin_name + KEYCLOAK_ADMIN_PASSWORD: /run/secrets/keycloak_admin_password KC_HOSTNAME_URL: "https://key.codemonsters.team" KC_HOSTNAME_STRICT_HTTPS: false KC_PROXY: edge @@ -29,10 +29,10 @@ volumes: keycloak-data: secrets: - keycloack_admin_name: - file: /opt/devops/keycloack/secrets/admin_name.txt - keycloack_admin_password: - file: /opt/devops/keycloack/secrets/admin_password.txt + keycloak_admin_name: + file: /opt/devops/keycloak/secrets/admin_name.txt + keycloak_admin_password: + file: /opt/devops/keycloak/secrets/admin_password.txt networks: intranet: \ No newline at end of file diff --git a/little-bastard/updatecert.sh b/little-bastard/updatecert.sh deleted file mode 100755 index eee745bcd24c4410fd5003248a4f20068dd3aa02..0000000000000000000000000000000000000000 --- a/little-bastard/updatecert.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -rsync -Lr /etc/letsencrypt/live/codemonsters.team /home/zero/certs/ -rsync -Lr /etc/letsencrypt/live/git.codemonsters.team /home/zero/certs/ -rsync -Lr /etc/letsencrypt/live/key.codemonsters.team /home/zero/certs/ -rsync -Lr /etc/letsencrypt/live/plantuml.codemonsters.team /home/zero/certs/ -chown -R zero:zero /home/zero/certs diff --git a/minikube/minikube-config-podman.sh b/minikube/minikube-config-podman.sh new file mode 100755 index 0000000000000000000000000000000000000000..b9db09723456132a19c62b8b4c75e8f693a68916 --- /dev/null +++ b/minikube/minikube-config-podman.sh @@ -0,0 +1,6 @@ +minikube config set rootless true +minikube config set profile k8s.codemonsters.team +minikube config set cpus 8 +minikube config set memory 16384 +minikube config view + diff --git a/minikube/minikube-config.sh b/minikube/minikube-config.sh index 8d370da91e31fd66a256f11be65d54a80408a945..e4a65aa08ac3633b8854f203ec50f40e2222e7fa 100755 --- a/minikube/minikube-config.sh +++ b/minikube/minikube-config.sh @@ -1,3 +1,4 @@ +minikube config set rootless false minikube config set profile k8s.codemonsters.team minikube config set cpus 8 minikube config set memory 16384 diff --git a/minikube/minikube-init-start-podman.sh b/minikube/minikube-init-start-podman.sh new file mode 100755 index 0000000000000000000000000000000000000000..d10bf9f0d506dec96988b99807b83098e75bcb98 --- /dev/null +++ b/minikube/minikube-init-start-podman.sh @@ -0,0 +1,5 @@ +# use this script to run when the system starts +minikube start --embed-certs \ +--apiserver-name k8s.codemonsters.team \ +--driver=podman \ +--container-runtime=containerd diff --git a/minikube/minikube-init-start.sh b/minikube/minikube-init-start.sh new file mode 100755 index 0000000000000000000000000000000000000000..2e6bcfc5cdf594f9e4c6d83c3d7a6343029fd0a2 --- /dev/null +++ b/minikube/minikube-init-start.sh @@ -0,0 +1,4 @@ +# use this script to run when the system starts +minikube start --embed-certs \ +--apiserver-name k8s.codemonsters.team \ +--container-runtime=docker diff --git a/minikube/minikube-start.sh b/minikube/minikube-start.sh index 357df86e5577379c160ee27a424e06b43485a410..62c69b8a85e855ea16d237eda6be97cd9a3026d9 100755 --- a/minikube/minikube-start.sh +++ b/minikube/minikube-start.sh @@ -1,4 +1,4 @@ # use this script to run when the system starts -docker compose -f /home/zero/core/nginx/docker-compose.yml stop \ +docker compose -f /var/opt/developer-sandbox/docker-compose.yml stop \ && minikube start --embed-certs --apiserver-name k8s.codemonsters.team --container-runtime=docker \ -&& docker compose -f /home/zero/core/nginx/docker-compose.yml up -d \ No newline at end of file +&& docker compose -f /var/opt/developer-sandbox/docker-compose.yml up -d diff --git a/nginx/docker-compose.yml b/nginx/docker-compose.yml index 95b18f3eb978b531928b2e09be6facfd5dc60818..f5cea34a1df68621465fe3b35abdb6f6fd0af55f 100644 --- a/nginx/docker-compose.yml +++ b/nginx/docker-compose.yml @@ -4,21 +4,15 @@ services: nginx: image: nginx:stable-alpine3.17 container_name: nginx - #restart: always -- first you need to start minicube, then start nginx - #look at the minicube launch script ../minikube/minikube-start.sh + #restart: always -- first you need to start minikube, then start nginx + #look at the minikube launch script ../minikube/minikube-start.sh hostname: nginx secrets: - - key-monsters-certificate-key - - key-monsters-certificate-fullchain - - git-monsters-certificate-key - - git-monsters-certificate-fullchain - monsters-certificate-key - monsters-certificate-fullchain - - plantuml-monsters-certificate-key - - plantuml-monsters-certificate-fullchain volumes: - type: bind - source: ./nginx + source: /var/opt/developer-sandbox/nginx/nginx target: /etc/nginx read_only: true - type: volume @@ -26,30 +20,20 @@ services: target: /var/logs/nginx read_only: false ports: - - "80:80" - "4443:4443" + - "8443:8443" networks: - k8s.codemonsters.team -volumes: - nginx-log: + - mechanicus-swarm secrets: - key-monsters-certificate-key: - file: /opt/devops/nginx/secrets/certs/key.codemonsters.team/privkey.pem - key-monsters-certificate-fullchain: - file: /opt/devops/nginx/secrets/certs/key.codemonsters.team/fullchain.pem - git-monsters-certificate-key: - file: /opt/devops/nginx/secrets/certs/git.codemonsters.team/privkey.pem - git-monsters-certificate-fullchain: - file: /opt/devops/nginx/secrets/certs/git.codemonsters.team/fullchain.pem monsters-certificate-key: - file: /opt/devops/nginx/secrets/certs/codemonsters.team/privkey.pem + file: /var/opt/developer-sandbox/data/certbot/etc/live/codemonsters.team/privkey.pem monsters-certificate-fullchain: - file: /opt/devops/nginx/secrets/certs/codemonsters.team/fullchain.pem - plantuml-monsters-certificate-key: - file: /opt/devops/nginx/secrets/certs/plantuml.codemonsters.team/privkey.pem - plantuml-monsters-certificate-fullchain: - file: /opt/devops/nginx/secrets/certs/plantuml.codemonsters.team/fullchain.pem - + file: /var/opt/developer-sandbox/data/certbot/etc/live/codemonsters.team/fullchain.pem networks: k8s.codemonsters.team: - external: true \ No newline at end of file + external: true + mechanicus-swarm: + +volumes: + nginx-log: diff --git a/nginx/nginx/nginx.conf b/nginx/nginx/nginx.conf index 2f5f2b691b551e7f1ae19994bf4a98ad7d161a48..108ba3d0cca45648da70ad29e778ff426b3c5ff7 100644 --- a/nginx/nginx/nginx.conf +++ b/nginx/nginx/nginx.conf @@ -13,22 +13,24 @@ http { sendfile on; tcp_nopush on; + #keycloak http flow + upstream keycloak { + server mechanicus:8888 weight=5; + } + #plantuml http flow upstream plantuml { - server 192.168.50.98:8001 weight=5; + server mechanicus:8001 weight=5; } #docker http flow upstream gitlab { - server 192.168.50.98:8000 weight=5; - } - #docker http flow - upstream keycloack { - server 192.168.50.98:8888 weight=5; + server mechanicus:8000 weight=5; } + # minikube ingress http flow upstream minikube { - server 192.168.49.2:80 weight=5; + server k8s.codemonsters.team:80 weight=5; } proxy_set_header Host $host; @@ -41,8 +43,8 @@ http { server { listen 4443 ssl; server_name plantuml.codemonsters.team; - ssl_certificate /run/secrets/plantuml-monsters-certificate-fullchain; - ssl_certificate_key /run/secrets/plantuml-monsters-certificate-key; + ssl_certificate /run/secrets/monsters-certificate-fullchain; + ssl_certificate_key /run/secrets/monsters-certificate-key; ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; ssl_ciphers HIGH:!aNULL:!MD5; location / { @@ -53,9 +55,10 @@ http { server { listen 4443 ssl; + client_max_body_size 512m; server_name git.codemonsters.team; - ssl_certificate /run/secrets/git-monsters-certificate-fullchain; - ssl_certificate_key /run/secrets/git-monsters-certificate-key; + ssl_certificate /run/secrets/monsters-certificate-fullchain; + ssl_certificate_key /run/secrets/monsters-certificate-key; ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; ssl_ciphers HIGH:!aNULL:!MD5; #redirect site to a new location @@ -72,12 +75,12 @@ http { server { listen 4443 ssl; server_name key.codemonsters.team; - ssl_certificate /run/secrets/key-monsters-certificate-fullchain; - ssl_certificate_key /run/secrets/key-monsters-certificate-key; + ssl_certificate /run/secrets/monsters-certificate-fullchain; + ssl_certificate_key /run/secrets/monsters-certificate-key; ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; ssl_ciphers HIGH:!aNULL:!MD5; location / { - proxy_pass http://keycloack; + proxy_pass http://keycloak; proxy_redirect off; } } @@ -94,5 +97,17 @@ http { proxy_redirect off; } } +} + +stream { + + upstream minikube-api { + server 192.168.49.2:8443 weight=5; + } + + server { + listen 8443; + proxy_pass minikube-api; + } } \ No newline at end of file diff --git a/research/init-developer-sandbox-docker.sh b/research/init-developer-sandbox-docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..cacb7cc896b3bb56f540c4cc7785f14ca4b19266 --- /dev/null +++ b/research/init-developer-sandbox-docker.sh @@ -0,0 +1,11 @@ +#!/bin/bash +#git clone https://git.codemonsters.team/devops/developer-sandbox.git /var/opt/developer-sandbox +#mkdir -p /var/opt/developer-sandbox/data/ +## adding k8s.mechanicus hosts +#sudo bash zero-server/modify-hosts.sh +# +/var/opt/developer-sandbox/minikube/minikube-config.sh +/var/opt/developer-sandbox/minikube/minikube-init-start.sh +/var/opt/developer-sandbox/minikube/minikube-enable-addons.sh +# +#docker compose -f /var/opt/developer-sandbox/docker-compose.yml diff --git a/research/init-developer-sandbox-podman.sh b/research/init-developer-sandbox-podman.sh new file mode 100755 index 0000000000000000000000000000000000000000..74aa2ed9887559b73a467a1c0405a7ec3f33a280 --- /dev/null +++ b/research/init-developer-sandbox-podman.sh @@ -0,0 +1,11 @@ +#!/bin/bash +#git clone https://git.codemonsters.team/devops/developer-sandbox.git /var/opt/developer-sandbox +#mkdir -p /var/opt/developer-sandbox/data/ +## adding k8s.mechanicus hosts +#sudo bash zero-server/modify-hosts.sh +# +/var/opt/developer-sandbox/minikube/minikube-config-podman.sh +/var/opt/developer-sandbox/minikube/minikube-init-start-podman.sh +/var/opt/developer-sandbox/minikube/minikube-enable-addons.sh +# +#docker compose -f /var/opt/developer-sandbox/docker-compose.yml \ No newline at end of file diff --git a/research/minikube-startup-trace-docker.txt b/research/minikube-startup-trace-docker.txt new file mode 100644 index 0000000000000000000000000000000000000000..638874b83800585a87b0f3e140077e1284692a01 --- /dev/null +++ b/research/minikube-startup-trace-docker.txt @@ -0,0 +1,40 @@ +% time seconds usecs/call calls errors syscall +------ ----------- ----------- --------- --------- ---------------- + 99.95 0.677185 112864 6 3 wait4 + 0.03 0.000217 72 3 clone + 0.01 0.000043 1 26 rt_sigprocmask + 0.00 0.000016 1 10 read + 0.00 0.000012 0 20 rt_sigaction + 0.00 0.000009 1 5 1 ioctl + 0.00 0.000008 2 3 rt_sigreturn + 0.00 0.000003 0 6 lseek + 0.00 0.000000 0 20 close + 0.00 0.000000 0 25 mmap + 0.00 0.000000 0 4 mprotect + 0.00 0.000000 0 1 munmap + 0.00 0.000000 0 3 brk + 0.00 0.000000 0 2 pread64 + 0.00 0.000000 0 1 1 access + 0.00 0.000000 0 1 dup2 + 0.00 0.000000 0 3 getpid + 0.00 0.000000 0 1 execve + 0.00 0.000000 0 1 uname + 0.00 0.000000 0 3 1 fcntl + 0.00 0.000000 0 1 sysinfo + 0.00 0.000000 0 1 getuid + 0.00 0.000000 0 1 getgid + 0.00 0.000000 0 1 geteuid + 0.00 0.000000 0 1 getegid + 0.00 0.000000 0 3 getppid + 0.00 0.000000 0 1 getpgrp + 0.00 0.000000 0 2 1 arch_prctl + 0.00 0.000000 0 1 futex + 0.00 0.000000 0 1 set_tid_address + 0.00 0.000000 0 33 13 openat + 0.00 0.000000 0 26 newfstatat + 0.00 0.000000 0 1 set_robust_list + 0.00 0.000000 0 3 prlimit64 + 0.00 0.000000 0 1 getrandom + 0.00 0.000000 0 1 rseq +------ ----------- ----------- --------- --------- ---------------- +100.00 0.677493 3051 222 20 total diff --git a/research/minikube-startup-trace-podman.txt b/research/minikube-startup-trace-podman.txt new file mode 100644 index 0000000000000000000000000000000000000000..25f8c73d679ba770ddb03c92db36f23581cf689d --- /dev/null +++ b/research/minikube-startup-trace-podman.txt @@ -0,0 +1,40 @@ +% time seconds usecs/call calls errors syscall +------ ----------- ----------- --------- --------- ---------------- +100.00 1.609212 268202 6 3 wait4 + 0.00 0.000000 0 11 read + 0.00 0.000000 0 20 close + 0.00 0.000000 0 6 lseek + 0.00 0.000000 0 25 mmap + 0.00 0.000000 0 4 mprotect + 0.00 0.000000 0 1 munmap + 0.00 0.000000 0 3 brk + 0.00 0.000000 0 20 rt_sigaction + 0.00 0.000000 0 26 rt_sigprocmask + 0.00 0.000000 0 3 rt_sigreturn + 0.00 0.000000 0 5 1 ioctl + 0.00 0.000000 0 2 pread64 + 0.00 0.000000 0 1 1 access + 0.00 0.000000 0 1 dup2 + 0.00 0.000000 0 3 getpid + 0.00 0.000000 0 3 clone + 0.00 0.000000 0 1 execve + 0.00 0.000000 0 1 uname + 0.00 0.000000 0 3 1 fcntl + 0.00 0.000000 0 1 sysinfo + 0.00 0.000000 0 1 getuid + 0.00 0.000000 0 1 getgid + 0.00 0.000000 0 1 geteuid + 0.00 0.000000 0 1 getegid + 0.00 0.000000 0 3 getppid + 0.00 0.000000 0 1 getpgrp + 0.00 0.000000 0 2 1 arch_prctl + 0.00 0.000000 0 1 futex + 0.00 0.000000 0 1 set_tid_address + 0.00 0.000000 0 33 13 openat + 0.00 0.000000 0 26 newfstatat + 0.00 0.000000 0 1 set_robust_list + 0.00 0.000000 0 3 prlimit64 + 0.00 0.000000 0 1 getrandom + 0.00 0.000000 0 1 rseq +------ ----------- ----------- --------- --------- ---------------- +100.00 1.609212 7216 223 20 total diff --git a/zero-server/bin/gitlab-backup.sh b/zero-server/bin/gitlab-backup.sh deleted file mode 100755 index 1230c6bbad429daf665c9f39e3147c2c7412a36c..0000000000000000000000000000000000000000 --- a/zero-server/bin/gitlab-backup.sh +++ /dev/null @@ -1 +0,0 @@ -rsync -azvP --delete /opt/devops/gitlab-data /home/zero/gitlab-backup diff --git a/zero-server/bin/restart-nginx.sh b/zero-server/bin/restart-nginx.sh deleted file mode 100644 index 482cb5ed28b26ba77c0cfdcc3f0f9f3e81c8bf76..0000000000000000000000000000000000000000 --- a/zero-server/bin/restart-nginx.sh +++ /dev/null @@ -1 +0,0 @@ -docker compose -f /home/zero/core/nginx/docker-compose.yml restart diff --git a/zero-server/bin/update-certs.sh b/zero-server/bin/update-certs.sh deleted file mode 100644 index 29eaf4f04b9f0e60e509d95e712860d2844f369f..0000000000000000000000000000000000000000 --- a/zero-server/bin/update-certs.sh +++ /dev/null @@ -1 +0,0 @@ -rsync -chavzP zero@little-bastard:/home/zero/certs/ /opt/devops/nginx/secrets/certs/ \ No newline at end of file diff --git a/zero-server/modify-hosts.sh b/zero-server/modify-hosts.sh new file mode 100755 index 0000000000000000000000000000000000000000..4dfb03b776d4d876af21096915ccc098dd5f6016 --- /dev/null +++ b/zero-server/modify-hosts.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# run it by sudo + +echo "192.168.49.2 k8s.codemonsters.team" >> /etc/hosts +echo "192.168.50.249 mechanicus" >> /etc/hosts \ No newline at end of file