Scaling et CI/CD du site Do-It

Ce script permet de réencoder les images en WebP, les vidéos en MP4 et les audios en Opus. Il permet également de mettre à jour les références des médias dans les fichiers HTML et de réécrire l'historique Git pour supprimer les médias originaux.

#!/bin/bash

if [[ "$1" == "--rewrite-git-history" ]]; then
    REWRITE_GIT_HISTORY=true
    shift
fi

if [[ "$1" == "-h" || "$1" == "--help" ]]; then
    echo "Usage: optimizer.bash "
    echo "Optimize media files in the current directory."
    exit 0
fi

temp_file=$(mktemp) # Temporary file to store paths for git filter-repo

if [[ "$1" == "image" || "$2" == "image" || "$3" == "image" ]]; then
    # Process image files (JPG, JPEG, PNG)
    find src -type f \( -name "*.jpg" -o -name "*.jpeg" -o -name "*.png" -o -name "*.JPG" -o -name "*.JPEG" -o -name "*.PNG" \) -print0 | while IFS= read -r -d '' img; do
        ffmpeg -y -i "$img" -q:v 75 "${img%.*}.webp" && \
        echo "$img" >> "$temp_file" && \
        rm "$img"

        if [[ "$REWRITE_GIT_HISTORY" == "true" ]]; then
             git rm --cached "$img"
        fi
    done

    # Update image references in HTML files
    find src -type f -name "*.md" -exec sed -i \
        -e '/http/! s/\.png/\.webp/g' \
        -e '/http/! s/\.PNG/\.webp/g' \
        -e '/http/! s/\.jpg/\.webp/g' \
        -e '/http/! s/\.JPG/\.webp/g' \
        -e '/http/! s/\.jpeg/\.webp/g' \
        -e '/http/! s/\.JPEG/\.webp/g' {} +

elif [[ "$1" == "video" || "$2" == "video" || "$3" == "video" ]]; then
    # Process video files (MP4, MOV, MKV)
    find src -type f \( -name "*.mp4" -o -name "*.mov" -o -name "*.mkv" \) -print0 | while IFS= read -r -d '' video; do
        ffmpeg -y -i "$video" -c:v libx264 -preset veryslow -c:a aac -b:a 96k "${video%.*}.compressed.mp4" && \
        echo "$video" >> "$temp_file" && \
        rm "$video" && \
        mv "${video%.*}.compressed.mp4" "${video%.*}.mp4"

        if [[ "$REWRITE_GIT_HISTORY" == "true" ]]; then
             git rm --cached "$video"
        fi
    done

    # Update video references in HTML files
    find src -type f -name "*.md" -exec sed -i \
        -e '/http/! s/\.mp4/\.compressed\.mp4/g' \
        -e '/http/! s/\.mov/\.compressed\.mp4/g' \
        -e '/http/! s/\.mkv/\.compressed\.mp4/g' {} +


elif [[ "$1" == "audio" || "$2" == "audio" || "$3" == "audio" ]]; then
    # Process audio files (WAV, FLAC, MP3, OGG)
    find src -type f \( -name "*.wav" -o -name "*.flac" -o -name "*.mp3" -o -name "*.ogg" \) -print0 | while IFS= read -r -d '' audio; do
        ffmpeg -y -i "$audio" -c:a libopus -b:a 96k "${audio%.*}.opus" && \
        echo "$audio" >> "$temp_file" && \
        rm "$audio"

        if [[ "$REWRITE_GIT_HISTORY" == "true" ]]; then
             git rm --cached "$audio"
        fi
    done

    # Update audio references in HTML files
    find src -type f -name "*.md" -exec sed -i \
        -e '/http/! s/\.wav/\.opus/g' \
        -e '/http/! s/\.flac/\.opus/g' \
        -e '/http/! s/\.mp3/\.opus/g' \
        -e '/http/! s/\.ogg/\.opus/g' {} +

else
    echo "Usage: optimizer.bash "
    exit 1
fi

# Rewrite git history for all stored paths
if [[ -s "$temp_file" && "$REWRITE_GIT_HISTORY" == "true" ]]; then
    git filter-repo --invert-paths --paths-from-file "$temp_file" --force
fi

# Cleanup temporary file
rm "$temp_file"

Ce script de build permet de réécrire les URLs des médias pour qu'ils pointent vers le serveur GitHub.\

if (process.env.NODE_ENV === "production") {

  // DO NOT REMOVE THOSE CONSTANTS,
  // PERFORMING A GIT REMOTE -V OPERATION ON EACH FILE IS TOO EXPENSIVE
  // HARDCODED VALUES ARE FINE IN OUR CONTEXT

  // Raw GitHub URL constants
  const RAW_GITHUB_BASE = "https://raw.githubusercontent.com";
  // Repositories owner
  const GITHUB_REPO_OWNER = "do-it-ecm";
  // Regex to match promo paths (and extract the promo year and relative path)
  const IS_PROMO_PATH = /src\/promos\/(\d{4}-\d{4})(\/?.*)?/;
  // Regex to match the CS paths (and extract the relative path)
  const IS_CS_PATH = /src\/cs\/(.*)/;
  // Commit ref to use for all submodules
  const COMMIT_REF = "refs/heads/main"; // Don't bother finding the commit ref for each submodule, just use main

  function mediaUrlTransform(context) {
    const urlsOptions = {
      eachURL: function (url, attr, tagName) {
        let remoteUrl = url;
        if (url.includes("://")) {
          // Don't transform external URLs
        } else { // Hopefully valid relative URL
          const baseDir = process.cwd();
          let absoluteDirPath = "";
          if (url.startsWith("/")) {
            url = url.slice(1);
            absoluteDirPath = path.dirname(path.resolve(baseDir, "src", url));
          } else {
            absoluteDirPath = path.dirname(path.resolve(baseDir, path.dirname(context.inputPath), url));
          }
          const promoMatch = absoluteDirPath.match(IS_PROMO_PATH);
          if (promoMatch) {
            // Promo path
            const promoYear = promoMatch[1];
            // Retrieve relative path (remove trailing slash)
            const relativePath = promoMatch[2] ? promoMatch[2].replace(/\/$/, "") : "";
            remoteUrl = `${RAW_GITHUB_BASE}/${GITHUB_REPO_OWNER}/promo-${promoYear}/${COMMIT_REF}/${relativePath}/${path.basename(url)}`;
          } else {
            const csMatch = absoluteDirPath.match(IS_CS_PATH);
            if (csMatch) {
              // CS path
              const relativePath = csMatch[1];
              remoteUrl = `${RAW_GITHUB_BASE}/${GITHUB_REPO_OWNER}/courses/${COMMIT_REF}/${relativePath}/${path.basename(url)}`;
            } else {
              const relativePath = path.relative(baseDir, absoluteDirPath);
              remoteUrl = `${RAW_GITHUB_BASE}/${GITHUB_REPO_OWNER}/do-it/${COMMIT_REF}/${relativePath}/${path.basename(url)}`;
            }
          }

        }
        return remoteUrl;
      },
      filter: {
        img: { src: true, srcset: true },
        video: { src: true, srcset: true },
        audio: { src: true },
        source: { src: true, srcset: true }
      }
    };
    return urls(urlsOptions);
  }
  eleventyConfig.htmlTransformer.addPosthtmlPlugin("html", mediaUrlTransform, { priority: -1 });
} else {
  console.warn("Skipping media URL transform in development mode");
}

Cette pipeline GitHub Actions utilise du caching pour accélérer le build et lance tous nos tests de conformité.
Elle ne se lance que lorsqu'une merge request est acceptée ou lorsqu'un submodule est mis à jour. Finito les push sur main!

name: check-compliance

on:
  repository_dispatch:
    types: [submodule-update]
  pull_request:
    types:
      - closed
    branches:
      - main

env:
  NODE_VERSION: "20.7" # Define the Node.js version here

jobs:
  check-compliance:
    runs-on: ubuntu-latest
    steps:
      # Checkout code without initializing submodules
      - uses: actions/checkout@v4
        with:
          submodules: false # Defer submodule initialization

      # Restore Git Submodules Cache
      - name: Restore Git Submodules Cache
        uses: actions/cache@v4
        with:
          path: .git/modules
          key: submodules-${{ github.ref_name }}
          restore-keys: |
            submodules-${{ github.ref_name }}
            submodules-

      # Git Submodule Update
      - name: Git Submodule Update
        run: |
          git submodule sync
          git submodule update --init --recursive
          git submodule foreach "git fetch origin main && git reset --hard origin/main"

      # Cache Git Submodules
      - name: Cache Git Submodules
        uses: actions/cache@v4
        with:
          path: .git/modules
          key: submodules-${{ github.ref_name }}

      # Restore Node.js Modules Cache
      - name: Restore Node.js Modules Cache
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: npm-cache-${{ env.NODE_VERSION }}-${{ github.ref_name }}
          restore-keys: |
            npm-cache-${{ env.NODE_VERSION }}-${{ github.ref_name }}
            npm-cache-${{ env.NODE_VERSION }}-
            npm-cache-

      # Use Node.js
      - name: Use Node.js ${{ env.NODE_VERSION }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      # Install dependencies
      - name: Install dependencies
        run: npm ci

      # Save Node.js Modules Cache
      - name: Save Node.js Modules Cache
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: npm-cache-${{ env.NODE_VERSION }}-${{ github.ref_name }}

      - name: Check Compliance
        run:  npm run check-compliance

Cette pipeline GitHub Actions construit le site Do-It à partir des submodules Git et applique les changements sur la branche gh-pages pour déployer le site. Elle ne conserve que la dernière version du build pour éviter de faire grossir le repository Git inutilement.

name: publish

on:
  workflow_run:
    workflows: ["check-compliance"]
    types:
      - completed

env:
  NODE_VERSION: "20.7" # Define the Node.js version here

jobs:
  publish:
    if: ${{ github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    steps:
      # Checkout code without initializing submodules
      - uses: actions/checkout@v4
        with:
          submodules: false # Defer submodule initialization

      # Restore Git Submodules Cache
      - name: Restore Git Submodules Cache
        uses: actions/cache@v4
        with:
          path: .git/modules
          key: submodules-${{ github.ref_name }}
          restore-keys: |
            submodules-${{ github.ref_name }}
            submodules-

      # Git Submodule Update
      - name: Git Submodule Update
        run: |
          git submodule sync
          git submodule update --init --recursive
          git submodule foreach "git fetch origin main && git reset --hard origin/main"

      # Cache Git Submodules
      - name: Cache Git Submodules
        uses: actions/cache@v4
        with:
          path: .git/modules
          key: submodules-${{ github.ref_name }}

      # Restore Node.js Modules Cache
      - name: Restore Node.js Modules Cache
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: npm-cache-${{ env.NODE_VERSION }}-${{ github.ref_name }}
          restore-keys: |
            npm-cache-${{ env.NODE_VERSION }}-${{ github.ref_name }}
            npm-cache-${{ env.NODE_VERSION }}-
            npm-cache-

      # Use Node.js
      - name: Use Node.js ${{ env.NODE_VERSION }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      # Install dependencies
      - name: Install dependencies
        run: npm ci

      # Save Node.js Modules Cache
      - name: Save Node.js Modules Cache
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: npm-cache-${{ env.NODE_VERSION }}-${{ github.ref_name }}

      - name: Build
        run: npm run build-github

      - name: Deploy
        uses: peaceiris/actions-gh-pages@v4
        with:
          publish_dir: dist
          github_token: ${{ secrets.GITHUB_TOKEN }}
          force_orphan: true
          publish_branch: gh-pages

Ce script permet d'exécuter d'une traite toutes les configurations pour la machine virtuelle (en supposant que le setup-alpine est déjà fait).

#!/bin/sh

set -e

apk update
apk upgrade
apk add --no-cache nginx git python3 openrc
rm -rf /var/cache/apk/*
mkdir -p /opt/do-it/jobs
echo "Creating the nginx configuration file in /etc/nginx/nginx.conf"
cat <<'EOF' > /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
pcre_jit on;
error_log /var/log/nginx/error.log warn;

events {
    worker_connections 1024;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    server_tokens off;
    client_max_body_size 1;

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    keepalive_requests 1000;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_session_cache shared:SSL:2m;
    ssl_session_timeout 1h;
    ssl_session_tickets off;

    gzip on;
    gzip_static on;
    gzip_comp_level 2;
    gzip_min_length 1000;
    gzip_types text/html text/css application/javascript;
    gzip_vary on;
    gzip_buffers 16 8k;

    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';

    access_log /var/log/nginx/access.log main;

    map $http_upgrade $connection_upgrade {
        default upgrade;
        '' close;
    }

    server {
        listen 80;
        listen [::]:80;

        root /opt/do-it/do-it;
        index index.html;

        location / {
            if ($request_method = POST) {
                proxy_pass http://localhost:3001;
                break;
            }

            try_files $uri $uri/ =404;
        }

        # Custom error page for 404
        error_page 404 /404.html;
        location = /404.html {
            internal;
        }

        # Cache HTML, CSS, JS for 1 week
        location ~* \.(html|css|js)$ {
            expires 7d;
            add_header Cache-Control "public, max-age=604800, must-revalidate";
            etag on;
        }
    }
}
EOF

echo "Creating the synchronizer script in /opt/do-it/jobs/synchronizer.sh"
cat <<'EOF' > /opt/do-it/jobs/synchronizer.sh
#!/bin/sh

DO_IT_PATH="/opt/do-it/do-it"
REMOTE_BRANCH="gh-pages"
LOG_DIRECTORY="/opt/do-it/logs"
LOG_FILE="${LOG_DIRECTORY}/synchronizer.log"

mkdir -p ${LOG_DIRECTORY}

# Function to prepend a formatted timestamp
log_with_timestamp() {
  while IFS= read -r line; do
    echo "$(date '+%Y/%m/%d %Hh%M:%S') $line"
  done
}

cd ${DO_IT_PATH}
GIT_ORIGIN_URL="$(git remote get-url origin)"
git fetch origin ${REMOTE_BRANCH} 2>&1 | log_with_timestamp | tee -a ${LOG_FILE}
git reset --hard origin/${REMOTE_BRANCH} 2>&1 | log_with_timestamp | tee -a ${LOG_FILE}
echo "Synchronized with the remote origin ${GIT_ORIGIN_URL} ${REMOTE_BRANCH} branch" 2>&1 | log_with_timestamp | tee -a ${LOG_FILE}
EOF

echo "Creating the server script in /opt/do-it/jobs/server.py"
cat <<'EOF' > /opt/do-it/jobs/server.py
# coding: utf-8
"""
Create a simple HTTP server that listens on a specified port.
The server receives POST requests, validates the token, and triggers a script.
Logs are written to a specified log file.
"""

from os import path, makedirs
import argparse
import logging
from http.server import HTTPServer, BaseHTTPRequestHandler
from subprocess import Popen, PIPE
from json import loads, JSONDecodeError
import hashlib
import hmac

class HTTPError(Exception):
    def __init__(self, status_code: int, detail: str):
        self.status_code = status_code
        self.detail = detail
        super().__init__(self.detail)

class WebhookHandler(BaseHTTPRequestHandler):
    def verify_signature(self, secret_token):
        """
        Verify the request payload's signature using the secret token.
        """
        signature_header = self.headers.get('X-Hub-Signature-256')
        payload_length = int(self.headers.get('Content-Length', 0))
        payload_body = self.rfile.read(payload_length).decode('utf-8')

        if secret_token:
            if not signature_header:
                raise HTTPError(401, "Missing signature header")

            hash_object = hmac.new(secret_token.encode('utf-8'), msg=payload_body.encode('utf-8'), digestmod=hashlib.sha256)
            expected_signature = "sha256=" + hash_object.hexdigest()
            if not hmac.compare_digest(expected_signature, signature_header):
                raise HTTPError(403, "Invalid request signature")

        try:
            parsed_body = loads(payload_body)
        except JSONDecodeError:
            raise HTTPError(400, "Invalid JSON payload")

        return parsed_body

    def do_POST(self):
        """
        Handle POST requests, validate payload, and trigger script execution.
        """
        try:
            # Verify the request signature and extract JSON payload
            request_body = self.verify_signature(self.server.secret_token)

            # Validate repository and branch
            repo_full_name = request_body['repository'].get('full_name')
            ref = request_body['ref']
            if repo_full_name != f"{self.server.github_repo_owner}/{self.server.github_repo}":
                raise HTTPError(400, f"Repository mismatch. Expected {self.server.github_repo_owner}/{self.server.github}")
            if ref != f"refs/heads/{self.server.github_branch}":
                raise HTTPError(400, f"Branch mismatch. Expected {self.server.github_branch}")

            # Run the update script if provided
            if self.server.update_script:
                self.run_script(self.server.update_script)

            self.send_response(200)
            self.end_headers()
            self.wfile.write(b"Webhook processed successfully.")
        except HTTPError as e:
            self.send_response(e.status_code)
            self.end_headers()
            self.wfile.write(e.detail.encode('utf-8'))
            logging.error(f"HTTPError: {e.detail}")
        except Exception as e:
            self.send_response(500)
            self.end_headers()
            self.wfile.write(f"Error: {e}".encode('utf-8'))
            logging.error(f"Unexpected error: {e}")

    @staticmethod
    def run_script(script_path):
        """
        Execute the update script.
        """
        logging.info(f"Executing script: {script_path}")
        process = Popen([script_path], stdout=PIPE, stderr=PIPE)
        stdout, stderr = process.communicate()
        if stderr:
            logging.error(f"Script error: {stderr.decode('utf-8')}")
            raise Exception(f"Script error: {stderr.decode('utf-8')}")
        logging.info(f"Script output: {stdout.decode('utf-8')}")
        logging.info("Script executed successfully")

    def log_message(self, format, *args):
        """
        Log an arbitrary message to the logger.
        """
        logging.info("%s - - [%s] %s\n" %
                     (self.client_address[0],
                      self.log_date_time_string(),
                      format % args))

def run_server(host, port, secret_token, update_script, github_repo_owner, github_repo, github_branch):
    """
    Start the web server on the specified host and port.
    """
    server = HTTPServer((host, port), WebhookHandler)
    server.secret_token = secret_token
    server.update_script = update_script
    server.github_repo_owner = github_repo_owner
    server.github_repo = github_repo
    server.github_branch = github_branch

    logging.info(f"Starting server on {host}:{port}")
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        server.shutdown()
        logging.info("Server stopped.")

def main():
    parser = argparse.ArgumentParser(description="Simple HTTP server to handle GitHub webhooks.")
    parser.add_argument('--host', default='localhost', help='Hostname to bind the server to (default: ::)')
    parser.add_argument('--port', type=int, default=3001, help='Port to listen on (default: 3001)')
    parser.add_argument('--secret-token', help='Secret token to validate incoming requests')
    parser.add_argument('--update-script', required=True, help='Path to the script to execute upon valid webhook')
    parser.add_argument('--github-repo-owner', required=True, help='GitHub repository owner')
    parser.add_argument('--github-repo', required=True, help='GitHub repository name')
    parser.add_argument('--github-branch', required=True, help='GitHub branch to monitor for webhooks')
    parser.add_argument('--log-file', required=True, help='Path to the log file (default: server.log)')

    args = parser.parse_args()

    # Create log directory if it does not exist
    log_dir = path.dirname(args.log_file)
    makedirs(log_dir, exist_ok=True)

    # Configure logging
    logging.basicConfig(
        filename=args.log_file,
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s'
    )

    run_server(
        host=args.host,
        port=args.port,
        secret_token=args.secret_token,
        update_script=args.update_script,
        github_repo_owner=args.github_repo_owner,
        github_repo=args.github_repo,
        github_branch=args.github_branch
    )

if __name__ == "__main__":
    main()
EOF

echo "Creating the init script in /etc/init.d/do-it-webserver"
cat <<'EOF' > /etc/init.d/do-it-webserver
#!/sbin/openrc-run
name="do-it-webserver"
description="Do-It Git Update Signals Python Web Server"

command="/usr/local/bin/do-it-webserver.sh"

pidfile="/var/run/${RC_SVCNAME}.pid"
logfile="/var/log/${RC_SVCNAME}.log"

depend() {
    need net
}

supervisor="supervise-daemon"
output_log="${logfile}"
error_log="${logfile}"
command_background="yes"
EOF

echo "Creating the web server service script in /usr/local/bin/do-it-webserver.sh"
cat <<'EOF' > /usr/local/bin/do-it-webserver.sh
#!/bin/sh
python3 /opt/do-it/jobs/server.py --update-script /opt/do-it/jobs/synchronizer.sh --github-repo-owner do-it-ecm --github-repo do-it --github-branch gh-pages --secret-token EXAMPLETOKEN --log-file /opt/do-it/logs/server.log --port 3001
EOF

echo "Cloning the do-it repository"
git clone --branch gh-pages --single-branch https://github.com/do-it-ecm/do-it.git /opt/do-it/do-it
echo "Setting permissions"
rc-update add nginx default
chmod +x /usr/local/bin/do-it-webserver.sh
chmod +x /etc/init.d/do-it-webserver
rc-update add do-it-webserver default
(crontab -l 2>/dev/null; echo "*/15 * * * * sh /opt/do-it/jobs/synchronizer.sh") | crontab -
sleep 5
echo "Starting the services"
rc-service nginx start
rc-service do-it-webserver start