Establish an e2e backend instance locally and in CI/CD

.github/workflows/run-ci-cd.yaml

@@ -176,6 +176,70 @@ jobs:
         run: |
           docker run -e DJANGO_SETTINGS_MODULE=settings.test --env-file backend/.env.example owasp/nest:test-backend-latest pytest
 
+  run-backend-e2e:
+    name: Run backend end-to-end tests
+    needs:
+      - scan-code
+      - scan-ci-dependencies
+    runs-on: ubuntu-latest
+    services:
+      db:
+        image: pgvector/pgvector:pg16
+        environment:
+          POSTGRES_DB: ${DJANGO_DB_NAME:-nest_db_e2e}
+          POSTGRES_PASSWORD: ${DJANGO_DB_PASSWORD:-nest_user_e2e_password}
+          POSTGRES_USER: ${DJANGO_DB_USER:-nest_user_e2e}
+        options: >-
+          --health-cmd="pg_isready -U ${DJANGO_DB_USER:-nest_user_e2e} -d ${DJANGO_DB_NAME:-nest_db_e2e} -h localhost -p 5432"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435
+
+      - name: Wait for database to be ready
+        run: |
+          until docker exec ${{ job.services.db.id }} pg_isready -U ${DJANGO_DB_USER:-nest_user_e2e} -d ${DJANGO_DB_NAME:-nest_db_e2e}; do
+            echo "Waiting for database..."
+            sleep 5
+          done
+
+      - name: Install PostgreSQL client
+        run: sudo apt-get install -y postgresql-client
+
+      - name: Build backend e2e test image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        with:
+          cache-from: |
+            type=gha
+          cache-to: |
+            type=gha,compression=zstd
+          context: backend
+          file: backend/docker/Dockerfile
+          load: true
+          platforms: linux/amd64
+          tags: owasp/nest:test-backend-e2e-latest
+
+      - name: Run migrations
+        run: |
+          docker run -e --env-file backend/.env.example owasp/nest:test-backend-e2e-latest python manage.py migrate
+
+      - name: Load Postgres data
+        env:
+          PGPASSWORD: ${DJANGO_DB_PASSWORD:-nest_user_e2e_password}
+        run: |
+          gunzip -c backend/data/nest-e2e-data.sql.gz | psql -h localhost -U ${DJANGO_DB_USER:-nest_user_e2e} -d ${DJANGO_DB_NAME:-nest_db_e2e}
+
+      - name: Run backend end-to-end tests
+        run: |
+          docker run -e DJANGO_SETTINGS_MODULE=settings.test --env-file backend/.env.example owasp/nest:test-backend-e2e-latest pytest
+
   run-frontend-unit-tests:
     name: Run frontend unit tests
     needs:

backend/Makefile

@@ -27,9 +27,16 @@ exec-backend-command:
 exec-backend-command-it:
 	@docker exec -it nest-backend $(CMD) 2>/dev/null
 
+exec-backend-e2e-command:
+	@docker exec -it nest-backend-e2e $(CMD)
+
+
 exec-db-command-it:
 	@docker exec -it nest-db $(CMD)
 
+exec-db-e2e-command:
+	@docker exec -it nest-db-e2e $(CMD)
+
 clear-cache:
 	@CMD="python manage.py clear_cache" $(MAKE) exec-backend-command
 
@@ -53,13 +60,18 @@ dump-data:
 	@CMD="sed -E -i 's/\"email\": *\"([^\"]|\\\")*\"/\"email\": \"\"/g' data/nest.json" $(MAKE) exec-backend-command
 	@CMD="gzip -f data/nest.json" $(MAKE) exec-backend-command
 
+dump-db-data-e2e:
+	@echo "Dumping Nest e2e data"
+	@CMD="pg_dumpall -U nest_user_e2e --clean | gzip -9 > backend/data/nest-e2e-data.sql.gz" $(MAKE) exec-db-e2e-command
+
 enrich-data: \
 	github-enrich-issues \
 	owasp-enrich-chapters \
 	owasp-enrich-committees \
 	owasp-enrich-events \
 	owasp-enrich-projects
 
+
 generate-sitemap:
 	@CMD="python manage.py generate_sitemap" $(MAKE) exec-backend-command
 
@@ -69,6 +81,10 @@ index-data:
 	@CMD="python manage.py algolia_update_replicas" $(MAKE) exec-backend-command
 	@CMD="python manage.py algolia_update_synonyms" $(MAKE) exec-backend-command
 
+load-data-e2e:
+	@echo "Loading Nest e2e data"
+	@CMD="python manage.py load_data" $(MAKE) exec-backend-e2e-command
+
 load-data:
 	@echo "Loading Nest data"
 	@CMD="python manage.py load_data" $(MAKE) exec-backend-command
@@ -96,6 +112,10 @@ restore-backup:
 	@echo "Restoring Nest backup"
 	@CMD="python manage.py restore_backup" $(MAKE) exec-backend-command
 
+run-backend-e2e:
+	@DOCKER_BUILDKIT=1 \
+	docker compose --project-name nest-e2e -f docker-compose/backend.e2e.yaml up --build --remove-orphans
+
 save-backup:
 	@echo "Saving Nest backup"
 	@CMD="python manage.py dumpdata --natural-primary --natural-foreign --indent=2" $(MAKE) exec-backend-command > backend/data/backup.json

backend/docker/Dockerfile

@@ -63,5 +63,3 @@ RUN rm -rf /home/owasp/.cache && \
     chmod +x /home/owasp/entrypoint.sh
 
 USER owasp
-
-CMD ["/home/owasp/entrypoint.sh"]

docker-compose/backend.e2e.yaml

@@ -0,0 +1,50 @@
+services:
+  backend:
+    container_name: nest-backend-e2e
+    command: >
+      sh -c '
+        python manage.py migrate &&
+        python manage.py runserver 0.0.0.0:8000
+      '
+    build:
+      context: ../backend
+      dockerfile: docker/Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+    env_file: ../backend/.env
+    environment:
+      DJANGO_DB_HOST: ${DJANGO_DB_HOST:-db}
+      DJANGO_DB_NAME: ${DJANGO_DB_NAME:-nest_db_e2e}
+      DJANGO_DB_PASSWORD: ${DJANGO_DB_PASSWORD:-nest_user_e2e_password}
+      DJANGO_DB_PORT: ${DJANGO_DB_PORT:-5432}
+      DJANGO_DB_USER: ${DJANGO_DB_USER:-nest_user_e2e}
+    networks:
+      - nest-network
+    ports:
+      - 8000:8000
+  db:
+    container_name: nest-db-e2e
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: ${DJANGO_DB_NAME:-nest_db_e2e}
+      POSTGRES_PASSWORD: ${DJANGO_DB_PASSWORD:-nest_user_e2e_password}
+      POSTGRES_USER: ${DJANGO_DB_USER:-nest_user_e2e}
+    healthcheck:
+      interval: 5s
+      retries: 5
+      test: [CMD-SHELL, pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB -h localhost -p 5432]
+      timeout: 5s
+    networks:
+      - nest-network
+    volumes:
+      - db-e2e-data:/var/lib/postgresql/data
+    ports:
+      - 5433:5432
+
+
+volumes:
+  db-e2e-data:
+
+networks:
+  nest-network:

docker-compose/production.yaml

@@ -1,6 +1,7 @@
 services:
   production-nest-backend:
     container_name: production-nest-backend
+    entrypoint: /home/owasp/entrypoint.sh
     image: owasp/nest:backend-production
     env_file: .env.backend
     depends_on:

docker-compose/staging.yaml

@@ -1,6 +1,7 @@
 services:
   staging-nest-backend:
     container_name: staging-nest-backend
+    entrypoint: /home/owasp/entrypoint.sh
     image: owasp/nest:backend-staging
     env_file: .env.backend
     depends_on: