From e6d4754682323dde061962478e66b791bcbd23d0 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Sun, 1 Jun 2025 14:13:40 -0700 Subject: [PATCH] Added Tinybird migrations (#2) no refs The current setup runs the base Ghost installation without Traffic Analytics functionality. This commit adds: - `tinybird-sync` service, which copies the latest Tinybird datafiles from the `ghost` container into a shared volume - `tinybird-deploy` service & Dockerfile that includes the `tb` CLI, and runs `tb --cloud deploy` on boot - Instructions for one-time manual setup of the Tinybird workspace in `TINYBIRD.md` After the one-time manual setup, this configuration should automatically update Tinybird's datasources and endpoints in sync with the Ghost container when it is updated. The initial setup is a bit clumsy and requires more manual steps than expected: - The tinybird datafiles are in the Ghost image, but we need to access them from the `tinybird-deploy` service, which includes the `tb` CLI. - When creating a new workspace in Tinybird, you can't access your admin token right away. Instead, it forces you to run `tb login` and `tb --cloud deploy` before you can access the rest of your workspace UI. This requires the user to install the `tb` CLI locally, and run an interactive login to authenticate with their Tinybird workspace. The generated `.tinyb` file is then mounted into the `tinybird-deploy` container, so this is only required for initial setup. - Ghost requires the Tinybird `stats` and `tracker` token to be provided at boot. This means the user has to manually copy these tokens (either from CLI or the Workspace UI) and add them to their `.env` file manually. - We may want to either publish the Docker image with the Tinybird CLI installed, or possibly add the `tb` CLI to the traffic-analytics container. --- .env.example | 9 ++++-- .gitignore | 1 + Caddyfile | 2 +- TINYBIRD.md | 14 ++++++++++ compose.yml | 67 ++++++++++++++++++++++++++++++++++++++++----- tinybird/Dockerfile | 14 ++++++++++ 6 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 TINYBIRD.md create mode 100644 tinybird/Dockerfile diff --git a/.env.example b/.env.example index 43a5e4f..a212343 100644 --- a/.env.example +++ b/.env.example @@ -4,5 +4,10 @@ DATABASE_ROOT_PASSWORD=reallysecurerootpassword DATABASE_USER=optionalusername DATABASE_PASSWORD=ghostpassword -TINYBIRD_ID=12345 -TINYBIRD_TOKEN=12345 +# Tinybird configuration for analytics +TINYBIRD_API_URL=https://api.tinybird.co +TINYBIRD_ID=745d3247-e887-4873-84f7-7414ec758b83 +TINYBIRD_TRACKER_TOKEN=p.eyJxxxxx +TINYBIRD_STATS_TOKEN=p.eyJxxxxx + +# COMPOSE_PROFILES=analytics,activitypub diff --git a/.gitignore b/.gitignore index 3ad396e..7857bf3 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ typings/ .DS_Store # Ghost Docker Custom +.tinyb diff --git a/Caddyfile b/Caddyfile index 8712708..7bb50bd 100644 --- a/Caddyfile +++ b/Caddyfile @@ -45,5 +45,5 @@ www.{$DOMAIN} { format console level INFO } - redir https://ghost.joegrigg.com{uri} + redir https://{$DOMAIN}{uri} } diff --git a/TINYBIRD.md b/TINYBIRD.md new file mode 100644 index 0000000..f30c56f --- /dev/null +++ b/TINYBIRD.md @@ -0,0 +1,14 @@ +# Tinybird Configuration + +Note: Currently Traffic Analytics features are behind a feature flag. For now, you'll need to enable developer experiments by setting `ENABLE_DEV_EXPERIMENTS=true` in your `.env` file, and enable the Traffic Analytics feature flag under Settings > Labs > Private Features. + +Steps: +1. Create a Tinybird account and a Tinybird workspace at [tinybird.co](https://auth.tinybird.co/login). You can select any cloud/region you choose. +2. Follow the first two steps in the Quickstart to install the Tinybird CLI and run `tb login`, but _do not proceed any further with the Quickstart instructions_. +3. Run `docker compose --profile=analytics up tinybird-sync`. This will copy the Tinybird files from the Ghost container into a shared volume. The service should log "Tinybird files synced into shared volume.", then exit. +4. Run `docker compose --profile=analytics up tinybird-deploy` and wait for the service to exit successfully. This will create your Tinybird datasources, pipes and API endpoints. It may take a minute or two to complete the first time. You should see "Deployment #1 is live!" in your terminal before the service exits. +5. Copy your Tinybird `stats_page` token: `tb --cloud token copy stats_page` and add it to your `.env` file as `TINYBIRD_STATS_TOKEN`. You can also copy the `stats_page` token from your Tinybird Workspace's UI. +6. Copy your Tinybird `tracker` token: `tb --cloud token copy tracker` and add it to your `.env` file as `TINYBIRD_TRACKER_TOKEN`. You can also copy the `tracker` token from your Tinybird Workspace's UI. +7. Find your workspace's events API endpoint: `tb --cloud info`, copy the value of "api", and add it to your `.env` file as `TINYBIRD_API_URL`. You can also find this value in your Tinybird Workspace's UI. +8. Run `docker compose --profile=analytics up -d` to start all services in the background. You can also set `COMPOSE_PROFILES=analytics` in your `.env` file to automatically include the `analytics` profile when running `docker compose` commands. +9. At this point, everything should be working. You can test it's working by visiting your site's homepage, then checking the Stats page in Ghost Admin — you should see a view recorded. diff --git a/compose.yml b/compose.yml index ac1fa93..2bfbfa1 100644 --- a/compose.yml +++ b/compose.yml @@ -30,16 +30,23 @@ services: database__connection__database: ghost enableDeveloperExperiments: true tinybird__tracker__endpoint: https://${DOMAIN:?DOMAIN environment variable is required}/.ghost/analytics/tb/web_analytics - tinybird__tracker__id: ${TINYBIRD_ID} + tinybird__tracker__id: ${TINYBIRD_ID:-} tinybird__tracker__datasource: analytics_events - tinybird__tracker__token: ${TINYBIRD_TOKEN} - tinybird__stats__endpoint: https://api.tinybird.co - tinybird__stats__id: ${TINYBIRD_ID} - tinybird__stats__token: ${TINYBIRD_TOKEN} + tinybird__tracker__token: ${TINYBIRD_TRACKER_TOKEN:-} + tinybird__stats__endpoint: ${TINYBIRD_API_URL:-https://api.tinybird.co} + tinybird__stats__id: ${TINYBIRD_ID:-} + tinybird__stats__token: ${TINYBIRD_STATS_TOKEN:-} volumes: - ghost_content:/var/lib/ghost/content depends_on: - - db + db: + condition: service_healthy + tinybird-sync: + condition: service_completed_successfully + required: false + tinybird-deploy: + condition: service_completed_successfully + required: false networks: - ghost_network @@ -57,6 +64,10 @@ services: volumes: - db_data:/var/lib/mysql - ./mysql-init:/docker-entrypoint-initdb.d + healthcheck: + test: mysql -u${DATABASE_USER:-ghost} -p${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} ghost -e 'select 1' + interval: 1s + retries: 120 networks: - ghost_network @@ -67,15 +78,57 @@ services: - "3000" environment: NODE_ENV: production - PROXY_TARGET: https://api.tinybird.co/v0/events + PROXY_TARGET: ${TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events + LOG_LEVEL: debug + profiles: [analytics] networks: - ghost_network + tinybird-sync: + image: ghost:5-alpine + command: > + sh -c " + if [ -d /var/lib/ghost/current/core/server/data/tinybird ]; then + rm -rf /shared/tinybird/*; + cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /shared/tinybird/; + echo 'Tinybird files synced into shared volume.'; + else + echo 'Tinybird source directory not found.'; + fi + " + volumes: + - tinybird_files:/shared/tinybird + networks: + - ghost_network + profiles: [analytics] + restart: no + + tinybird-deploy: + build: + context: ./tinybird + dockerfile: Dockerfile + working_dir: /home/tinybird + command: > + sh -c " + tb --cloud deploy + " + volumes: + - .tinyb:/home/tinybird/.tinyb + - tinybird_files:/home/tinybird + depends_on: + tinybird-sync: + condition: service_completed_successfully + profiles: [analytics] + networks: + - ghost_network + tty: true + volumes: ghost_content: db_data: caddy_data: caddy_config: + tinybird_files: networks: ghost_network: diff --git a/tinybird/Dockerfile b/tinybird/Dockerfile new file mode 100644 index 0000000..3153238 --- /dev/null +++ b/tinybird/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.13-slim + +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /home/tinybird + +# Install Tinybird using the standard installation script +RUN curl https://tinybird.co | sh + +ENV PATH="/root/.local/bin:$PATH"