From 107774c831057b30c2d6d9f62531581387867d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Manchon?= Date: Wed, 3 Jun 2026 11:05:56 +0200 Subject: [PATCH 1/6] templating --- .github/workflows/build.yml | 7 +- .rubocop.yml | 6 + .ruby-version | 2 +- Gemfile | 5 + Gemfile.lock | 29 +++++ LINUX.md | 41 ++++++- LINUX_keep_current.md | 25 +++- WINDOWS.md | 39 +++++- WINDOWS_keep_current.md | 25 +++- _partials/es/keep_current.md | 3 +- _partials/gcp_cli_oauth.md | 8 +- _partials/gcp_cli_setup.md | 12 +- _partials/keep_current.md | 59 +++++---- _partials/python_checkup.md | 6 +- _partials/terraform.md | 24 ++-- _partials/terraform_vm.md | 48 ++++---- _partials/vscode_ssh_connection.md | 4 +- build.rb | 185 +++-------------------------- builds/LINUX.yml | 29 +++++ builds/LINUX_keep_current.yml | 6 + builds/WINDOWS.yml | 28 +++++ builds/WINDOWS_keep_current.yml | 6 + builds/macOS.yml | 29 +++++ builds/macOS_keep_current.yml | 6 + constants/constants.yml | 7 ++ constants/repos.yml | 6 + doc/README.md | 26 ++++ lib/build_spec.rb | 18 +++ lib/builder.rb | 19 +++ lib/generator.rb | 27 +++++ lib/localized_build.rb | 18 +++ lib/partial.rb | 19 +++ lib/partial/local.rb | 16 +++ lib/partial/remote.rb | 30 +++++ lib/partial_cache.rb | 17 +++ macOS.md | 39 +++++- macOS_keep_current.md | 25 +++- 37 files changed, 623 insertions(+), 276 deletions(-) create mode 100644 .rubocop.yml create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 builds/LINUX.yml create mode 100644 builds/LINUX_keep_current.yml create mode 100644 builds/WINDOWS.yml create mode 100644 builds/WINDOWS_keep_current.yml create mode 100644 builds/macOS.yml create mode 100644 builds/macOS_keep_current.yml create mode 100644 constants/constants.yml create mode 100644 constants/repos.yml create mode 100644 doc/README.md create mode 100644 lib/build_spec.rb create mode 100644 lib/builder.rb create mode 100644 lib/generator.rb create mode 100644 lib/localized_build.rb create mode 100644 lib/partial.rb create mode 100644 lib/partial/local.rb create mode 100644 lib/partial/remote.rb create mode 100644 lib/partial_cache.rb diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef8eeef..e6b731f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,11 +6,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - name: Install dependencies + run: bundle install - name: Build run: | git config user.name github-actions git config user.email github-actions@github.com - ruby build.rb + bundle exec ruby build.rb if ! git diff --exit-code then git add . diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..6393cf2 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,6 @@ + + Layout/HashAlignment: + Enabled: false + + Layout/LeadingEmptyLines: + Enabled: false diff --git a/.ruby-version b/.ruby-version index ef538c2..7636e75 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.1.2 +4.0.5 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..632fced --- /dev/null +++ b/Gemfile @@ -0,0 +1,5 @@ +source "https://rubygems.org" + +gem "liquid" +gem "base64" +gem "cgi" diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..29d32bb --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,29 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + bigdecimal (4.1.2) + cgi (0.5.1) + liquid (5.12.0) + bigdecimal + strscan (>= 3.1.1) + strscan (3.1.8) + +PLATFORMS + arm64-darwin-24 + ruby + +DEPENDENCIES + base64 + cgi + liquid + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd + cgi (0.5.1) sha256=e93fcafc69b8a934fe1e6146121fa35430efa8b4a4047c4893764067036f18e9 + liquid (5.12.0) sha256=5a3c2c2430cd925d21c53e4ed9abea52cd0a9da53b541422f81dee79aca2a673 + strscan (3.1.8) sha256=aae2db611a225559f21ffbb71765c9a4e60fd262534a9ea84f4f11c7f32f679e + +BUNDLED WITH + 4.0.10 diff --git a/LINUX.md b/LINUX.md index c3d40f3..81a6f48 100644 --- a/LINUX.md +++ b/LINUX.md @@ -326,6 +326,9 @@ The `gcloud` Command Line Interface (CLI) is used to communicate with Google Clo + + + Add the `APT` repository and install with: ```bash @@ -345,6 +348,7 @@ gcloud --version + ### Authenticate gcloud We need to authenticate the `gcloud` CLI tool and set the project so it can interact with Google from the terminal. @@ -367,9 +371,12 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog + + For pasting into the terminal, your might need to use `ctrl + shift + v` + You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: @@ -419,6 +426,9 @@ Terraform is a tool for [Infrastructure as Code (IaC) 🔗](https://en.wikipedia + + + Install some system requirements requirements: ```bash sudo apt-get update && sudo apt-get install -y gnupg software-properties-common @@ -450,6 +460,7 @@ sudo apt-get install terraform ``` + Verify the installation with: ```bash @@ -462,9 +473,13 @@ The output should look similar to: Terraform v1.14.3 on _ + + + # Linux example # Terraform v1.14.3 # on linux_amd64 + ``` @@ -509,6 +524,9 @@ First we'll create a folder and download the terraform files with: + + + ```bash mkdir -p ~/code/wagon-de-bootcamp ``` @@ -521,14 +539,19 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le ``` + ### Set variables + + + Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. + It should look like: ```bash @@ -544,8 +567,12 @@ We'll need to change some values in this file. Here's were you can find the requ - **region:** take a look at the GCP Region and Zone documentation at this [link here](https://cloud.google.com/compute/docs/regions-zones#available). We generally recommend you choose a geographically nearby region. - **zone:** Zone is a subset of region. it is almost always the same as **region** appended with `-a`, `-b`, or `-c`. The zone you select within a region should not have a functional impact. - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. + + + - **instance_user:** in your terminal, run `whoami`, and enter the value + After completing this file, it might look similar to: ```bash @@ -559,7 +586,11 @@ instance_user = "taylorswift" Make sure to save the `terraform.tfvars` file, navigate into the directory with the terraform files using your terminal with: ```bash + + + cd ~/code/wagon-de-bootcamp + ``` Initialise and test the terraform config files with: @@ -624,6 +655,7 @@ For example, try running: ``` + ### Connect with VS Code To connect to your Virtual Machine, click on the small symbol at the very bottom-left corner of VS Code: @@ -698,9 +730,12 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog + + For pasting into the terminal, your might need to use `ctrl + shift + v` + You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: @@ -823,7 +858,7 @@ We will use the GitHub CLI (`gh`) to connect to GitHub using *SSH*, a protocol t First in order to **login**, copy-paste the following command in your terminal: -:warning: **DO NOT edit the `email`** +:warning: **DO NOT edit the `email`** — Even though `user:email` looks like a placeholder for your actual email address, it isn't — do not replace it. ```bash gh auth login -s 'user:email' -w --git-protocol ssh @@ -835,7 +870,9 @@ gh auth login -s 'user:email' -w --git-protocol ssh If you already have SSH keys, you will see instead `Upload your SSH public key to your GitHub account?` With the arrows, select your public key file path and press `Enter`. -- `Enter a passphrase for your new SSH key (Optional)`. Type something you want and that you'll remember. It's a password to protect your private key stored on your hard drive. Then press `Enter`. +- `Enter a passphrase for your new SSH key (Optional)`: + - **FOR MOST PEOPLE:** Just press `Enter` to skip. You don't need a passphrase for the bootcamp and it would prompt you every time you use the key. There is a risk, however, that if someone steals your laptop, they could then push to GitHub. + - **IF SECURITY IS REALLY IMPORTANT TO YOU:** Enter a passphrase of your choice and press `Enter`. It's _really_ important that if you enter a passphrase, you write it down somewhere immediately and do not lose/forget it. You will need to enter this frequently. - `Title for your SSH key`. You can leave it at the proposed "GitHub CLI", press `Enter`. diff --git a/LINUX_keep_current.md b/LINUX_keep_current.md index 067667d..56a19c6 100644 --- a/LINUX_keep_current.md +++ b/LINUX_keep_current.md @@ -86,10 +86,14 @@ type -a pyenv > /dev/null && eval "$(pyenv init --path)" Update pyenv : + + ``` bash cd $(pyenv root) && git pull ``` + + Install the current python version : ```bash @@ -140,10 +144,14 @@ pyenv versions pip install -U pip ``` + + + ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` + ## GCP Make sure that the `gcloud` command is linked to the email address of your Google Cloud Platform account : @@ -212,8 +220,7 @@ cat $GOOGLE_APPLICATION_CREDENTIALS { "type": "service_account", "project_id": "your-gcp-project-id", - "private_key_id": "a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInM=\n-----END PRIVATE KEY-----\n", + "private_key_id": "...", "client_email": "your-service-account@your-service-account.iam.gserviceaccount.com", "client_id": "105410541054105410541", "auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -252,12 +259,16 @@ gcloud auth configure-docker ## Docker + + Start Docker : ``` bash sudo service docker start ``` + + Verify that Docker can run the hello-world image : ``` bash @@ -266,6 +277,8 @@ docker run hello-world 👉 Make sure that this command completes correctly + + Start Docker : ``` bash @@ -273,21 +286,23 @@ sudo service docker stop ``` + + ### Python setup check up Check your Python version with the following commands: ```bash -zsh -c "$(curl -fsSL )" 3.8.12 +zsh -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/python_checker.sh)" 3.8.12 ``` Run the following command to check if you successfully installed the required packages: ```bash -zsh -c "$(curl -fsSL )" +zsh -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.sh)" ``` Now run the following command to check if you can load these packages: ```bash -python -c "$(curl -fsSL )" +python -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.py)" ``` Make sure you can run Jupyter: diff --git a/WINDOWS.md b/WINDOWS.md index 381eeb9..c4bf7cc 100644 --- a/WINDOWS.md +++ b/WINDOWS.md @@ -292,6 +292,8 @@ The `gcloud` Command Line Interface (CLI) is used to communicate with Google Clo + + To install, download the Google Cloud CLI installer from this [link here 🔗](https://cloud.google.com/sdk/docs/install#windows). Once it's finished downloading, launch the installer and follow the prompts. You only need to install `gcloud` for the current user. @@ -326,6 +328,8 @@ Now `gcloud` is installed and authenticated 🚀 + + ### Application Default Credentials Application Default Credentials are for authenticating our **code** (Terraform and Python 🐍) to interact with Google services and resources. It's a small distinction between `gcloud` and **code**, but an important one. @@ -345,6 +349,8 @@ Terraform is a tool for [Infrastructure as Code (IaC) 🔗](https://en.wikipedia + + ### Download To install terraform, download the **zip archive** from the Terraform install page at this [link here 🔗](https://developer.hashicorp.com/terraform/install). @@ -385,6 +391,8 @@ To update your path: + + Verify the installation with: ```bash @@ -397,9 +405,13 @@ The output should look similar to: Terraform v1.14.3 on _ + + # Windows example # Terraform v1.14.3 # on windows_amd64 + + ``` @@ -443,6 +455,8 @@ First we'll create a folder and download the terraform files with: + + Using the **Command Prompt** (cmd), run the following: ❗ Note: The below commands will only work in **Command Prompt** - they will not work in Windows Powershell. @@ -460,14 +474,20 @@ curl -L -o "%USERPROFILE%\wagon-de-bootcamp\.terraform.lock.hcl" https://raw.git + + ### Set variables + + Open up the file `C:\Users\\wagon-de-bootcamp\terraform.tfvars` in VS Code or any other code editor. + + It should look like: ```bash @@ -483,8 +503,12 @@ We'll need to change some values in this file. Here's were you can find the requ - **region:** take a look at the GCP Region and Zone documentation at this [link here](https://cloud.google.com/compute/docs/regions-zones#available). We generally recommend you choose a geographically nearby region. - **zone:** Zone is a subset of region. it is almost always the same as **region** appended with `-a`, `-b`, or `-c`. The zone you select within a region should not have a functional impact. - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. + + - **instance_user:** in Command Prompt, run `echo %username%`, and enter the value - try and remember your username, you will need it later on + + After completing this file, it might look similar to: ```bash @@ -498,7 +522,11 @@ instance_user = "taylorswift" Make sure to save the `terraform.tfvars` file, navigate into the directory with the terraform files using your terminal with: ```bash + + cd %USERPROFILE%\wagon-de-bootcamp + + ``` Initialise and test the terraform config files with: @@ -562,6 +590,7 @@ For example, try running: # $ ssh lw-de-vm-.. ``` + ### Confirm Your SSH Settings Let's take a look at the SSH configuration that was just created and verify it. In VS Code: @@ -634,6 +663,7 @@ icacls %USERPROFILE%\.ssh\google_compute_engine /grant:r SYSTEM:(R) && ^ icacls %USERPROFILE%\.ssh\google_compute_engine ``` + ### Connect with VS Code To connect to your Virtual Machine, click on the small symbol at the very bottom-left corner of VS Code: @@ -707,10 +737,13 @@ It's is usually the first check box. We recommend allowing **Google Auth Library** to: _View and sign in to your Google Cloud SQL instances._ + For pasting into the terminal, your might need to use `ctrl + shift + v` + + You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: @@ -833,7 +866,7 @@ We will use the GitHub CLI (`gh`) to connect to GitHub using *SSH*, a protocol t First in order to **login**, copy-paste the following command in your terminal: -:warning: **DO NOT edit the `email`** +:warning: **DO NOT edit the `email`** — Even though `user:email` looks like a placeholder for your actual email address, it isn't — do not replace it. ```bash gh auth login -s 'user:email' -w --git-protocol ssh @@ -845,7 +878,9 @@ gh auth login -s 'user:email' -w --git-protocol ssh If you already have SSH keys, you will see instead `Upload your SSH public key to your GitHub account?` With the arrows, select your public key file path and press `Enter`. -- `Enter a passphrase for your new SSH key (Optional)`. Type something you want and that you'll remember. It's a password to protect your private key stored on your hard drive. Then press `Enter`. +- `Enter a passphrase for your new SSH key (Optional)`: + - **FOR MOST PEOPLE:** Just press `Enter` to skip. You don't need a passphrase for the bootcamp and it would prompt you every time you use the key. There is a risk, however, that if someone steals your laptop, they could then push to GitHub. + - **IF SECURITY IS REALLY IMPORTANT TO YOU:** Enter a passphrase of your choice and press `Enter`. It's _really_ important that if you enter a passphrase, you write it down somewhere immediately and do not lose/forget it. You will need to enter this frequently. - `Title for your SSH key`. You can leave it at the proposed "GitHub CLI", press `Enter`. diff --git a/WINDOWS_keep_current.md b/WINDOWS_keep_current.md index 3a687df..f139e8c 100644 --- a/WINDOWS_keep_current.md +++ b/WINDOWS_keep_current.md @@ -86,10 +86,14 @@ type -a pyenv > /dev/null && eval "$(pyenv init --path)" Update pyenv : + + + ``` bash cd $(pyenv root) && git pull ``` + Install the current python version : ```bash @@ -140,10 +144,14 @@ pyenv versions pip install -U pip ``` + + ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` + + ## GCP Make sure that the `gcloud` command is linked to the email address of your Google Cloud Platform account : @@ -212,8 +220,7 @@ cat $GOOGLE_APPLICATION_CREDENTIALS { "type": "service_account", "project_id": "your-gcp-project-id", - "private_key_id": "a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInM=\n-----END PRIVATE KEY-----\n", + "private_key_id": "...", "client_email": "your-service-account@your-service-account.iam.gserviceaccount.com", "client_id": "105410541054105410541", "auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -252,12 +259,16 @@ gcloud auth configure-docker ## Docker + + + Start Docker : ``` bash sudo service docker start ``` + Verify that Docker can run the hello-world image : ``` bash @@ -266,6 +277,9 @@ docker run hello-world 👉 Make sure that this command completes correctly + + + Start Docker : ``` bash @@ -281,21 +295,22 @@ wsl --shutdown If the command still does not work, try to restart your Windows machine + ### Python setup check up Check your Python version with the following commands: ```bash -zsh -c "$(curl -fsSL )" 3.8.12 +zsh -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/python_checker.sh)" 3.8.12 ``` Run the following command to check if you successfully installed the required packages: ```bash -zsh -c "$(curl -fsSL )" +zsh -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.sh)" ``` Now run the following command to check if you can load these packages: ```bash -python -c "$(curl -fsSL )" +python -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.py)" ``` Make sure you can run Jupyter: diff --git a/_partials/es/keep_current.md b/_partials/es/keep_current.md index 5053334..20a4f6a 100644 --- a/_partials/es/keep_current.md +++ b/_partials/es/keep_current.md @@ -252,8 +252,7 @@ cat $GOOGLE_APPLICATION_CREDENTIALS { "type": "service_account", "project_id": "your-gcp-project-id", - "private_key_id": "a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInM=\n-----END PRIVATE KEY-----\n", + "private_key_id": "...", "client_email": "your-service-account@your-service-account.iam.gserviceaccount.com", "client_id": "105410541054105410541", "auth_uri": "https://accounts.google.com/o/oauth2/auth", diff --git a/_partials/gcp_cli_oauth.md b/_partials/gcp_cli_oauth.md index 94caf50..1f6b972 100644 --- a/_partials/gcp_cli_oauth.md +++ b/_partials/gcp_cli_oauth.md @@ -18,17 +18,17 @@ It's is usually the first check box. We recommend allowing **Google Auth Library** to: _View and sign in to your Google Cloud SQL instances._ -$WINDOWS_START +{% if os == "windows" %} For pasting into the terminal, your might need to use `ctrl + shift + v` -$WINDOWS_END +{% endif %} -$LINUX_START +{% if os == "linux" %} For pasting into the terminal, your might need to use `ctrl + shift + v` -$LINUX_END +{% endif %} You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. diff --git a/_partials/gcp_cli_setup.md b/_partials/gcp_cli_setup.md index 50c5304..3040710 100644 --- a/_partials/gcp_cli_setup.md +++ b/_partials/gcp_cli_setup.md @@ -4,7 +4,7 @@ The `gcloud` Command Line Interface (CLI) is used to communicate with Google Clo ### Install gcloud -$MAC_START +{% if os == "macos" %} #### 1. Check to see if python 3 is available, otherwise install @@ -81,9 +81,9 @@ Once the installer is finished, test the install by running: gcloud --version ``` -$MAC_END +{% endif %} -$WINDOWS_START +{% if os == "windows" %} To install, download the Google Cloud CLI installer from this [link here 🔗](https://cloud.google.com/sdk/docs/install#windows). @@ -116,9 +116,9 @@ Your active configurations: [default] Now `gcloud` is installed and authenticated 🚀 -$WINDOWS_END +{% endif %} -$LINUX_START +{% if os == "linux" %} Add the `APT` repository and install with: @@ -137,4 +137,4 @@ gcloud --version ``` 👉 [Install documentation 🔗](https://cloud.google.com/sdk/docs/install#deb) -$LINUX_END +{% endif %} diff --git a/_partials/keep_current.md b/_partials/keep_current.md index df45132..93e13d5 100644 --- a/_partials/keep_current.md +++ b/_partials/keep_current.md @@ -86,26 +86,26 @@ type -a pyenv > /dev/null && eval "$(pyenv init --path)" Update pyenv : -$MAC_START +{% if os == "macos" %} ``` bash brew update && brew upgrade pyenv ``` -$MAC_END -$LINUX_START +{% endif %} +{% if os == "linux" %} ``` bash cd $(pyenv root) && git pull ``` -$LINUX_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} ``` bash cd $(pyenv root) && git pull ``` -$WINDOWS_END +{% endif %} Install the current python version : ```bash -pyenv install +pyenv install {{ PYTHON_VERSION }} ``` 👉 Make sure that the command completes correctly and **restart your terminal** @@ -119,7 +119,7 @@ pyenv virtualenv-delete lewagon_current Create a new virtual environment : ```bash -pyenv virtualenv lewagon_current +pyenv virtualenv {{ PYTHON_VERSION }} lewagon_current ``` Set the new virtual environment as default : @@ -138,8 +138,8 @@ pyenv versions ``` bash system - - /envs/lewagon_current + {{ PYTHON_VERSION }} + {{ PYTHON_VERSION }}/envs/lewagon_current 3.7.6 3.7.6/envs/lewagon * lewagon_current @@ -152,7 +152,7 @@ pyenv versions pip install -U pip ``` -$MAC_START +{% if os == "macos" %} If your computer uses **Apple Silicon**, expand the paragraph below and go through it. Otherwise ignore it.
@@ -172,17 +172,17 @@ If your computer uses **Apple Intel**, expand the paragraph below and go through pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/apple_intel.txt ```
-$MAC_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` -$WINDOWS_END -$LINUX_START +{% endif %} +{% if os == "linux" %} ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` -$LINUX_END +{% endif %} ## GCP @@ -252,8 +252,7 @@ cat $GOOGLE_APPLICATION_CREDENTIALS { "type": "service_account", "project_id": "your-gcp-project-id", - "private_key_id": "a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInM=\n-----END PRIVATE KEY-----\n", + "private_key_id": "...", "client_email": "your-service-account@your-service-account.iam.gserviceaccount.com", "client_id": "105410541054105410541", "auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -292,23 +291,23 @@ gcloud auth configure-docker ## Docker -$MAC_START +{% if os == "macos" %} Start the Docker app -$MAC_END -$LINUX_START +{% endif %} +{% if os == "linux" %} Start Docker : ``` bash sudo service docker start ``` -$LINUX_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} Start Docker : ``` bash sudo service docker start ``` -$WINDOWS_END +{% endif %} Verify that Docker can run the hello-world image : @@ -318,17 +317,17 @@ docker run hello-world 👉 Make sure that this command completes correctly -$MAC_START +{% if os == "macos" %} Stop the Docker app -$MAC_END -$LINUX_START +{% endif %} +{% if os == "linux" %} Start Docker : ``` bash sudo service docker stop ``` -$LINUX_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} Start Docker : ``` bash @@ -342,4 +341,4 @@ wsl --shutdown ``` If the command still does not work, try to restart your Windows machine -$WINDOWS_END +{% endif %} diff --git a/_partials/python_checkup.md b/_partials/python_checkup.md index 363c219..9299275 100644 --- a/_partials/python_checkup.md +++ b/_partials/python_checkup.md @@ -2,17 +2,17 @@ Check your Python version with the following commands: ```bash -zsh -c "$(curl -fsSL )" +zsh -c "$(curl -fsSL {{ PYTHON_CHECKER_URL }})" {{ PYTHON_VERSION }} ``` Run the following command to check if you successfully installed the required packages: ```bash -zsh -c "$(curl -fsSL )" +zsh -c "$(curl -fsSL {{ PIP_CHECKER_URL }})" ``` Now run the following command to check if you can load these packages: ```bash -python -c "$(curl -fsSL )" +python -c "$(curl -fsSL {{ PIP_LOADER_URL }})" ``` Make sure you can run Jupyter: diff --git a/_partials/terraform.md b/_partials/terraform.md index 6da4d87..6b7bc81 100644 --- a/_partials/terraform.md +++ b/_partials/terraform.md @@ -2,7 +2,7 @@ Terraform is a tool for [Infrastructure as Code (IaC) 🔗](https://en.wikipedia.org/wiki/Infrastructure_as_code) to create, destroy, and manage resources in the cloud. -$MAC_START +{% if os == "macos" %} To install Terraform, navigate to the [Terraform downloads page 🔗](https://developer.hashicorp.com/terraform/install): - Under **Binary download**, select the binary that corresponds to your CPU architecture: @@ -15,9 +15,9 @@ To install Terraform, navigate to the [Terraform downloads page 🔗](https://de sudo cp ~/Downloads/terraform_*/terraform /usr/local/bin/ ``` -$MAC_END +{% endif %} -$WINDOWS_START +{% if os == "windows" %} ### Download @@ -57,9 +57,9 @@ To update your path: 6. Close **Command Prompt** and open it again -$WINDOWS_END +{% endif %} -$LINUX_START +{% if os == "linux" %} Install some system requirements requirements: ```bash @@ -91,7 +91,7 @@ sudo apt update sudo apt-get install terraform ``` -$LINUX_END +{% endif %} Verify the installation with: @@ -105,19 +105,19 @@ The output should look similar to: Terraform v1.14.3 on _ -$MAC_START +{% if os == "macos" %} # macOS (M chip) example # Terraform 1.14.3 # on darwin_arm64 -$MAC_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} # Windows example # Terraform v1.14.3 # on windows_amd64 -$WINDOWS_END -$LINUX_START +{% endif %} +{% if os == "linux" %} # Linux example # Terraform v1.14.3 # on linux_amd64 -$LINUX_END +{% endif %} ``` diff --git a/_partials/terraform_vm.md b/_partials/terraform_vm.md index 5df0caf..76a8523 100644 --- a/_partials/terraform_vm.md +++ b/_partials/terraform_vm.md @@ -36,7 +36,7 @@ We almost have all the necessary parts to create your VM using **terraform**. We First we'll create a folder and download the terraform files with: -$MAC_START +{% if os == "macos" %} In a terminal, run the following commands: @@ -51,9 +51,9 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le && curl -L -o ~/code/wagon-de-bootcamp/.terraform.lock.hcl https://raw.githubusercontent.com/lewagon/data-engineering-setup/main-automation/automation/infra/.terraform.lock.hcl ``` -$MAC_END +{% endif %} -$WINDOWS_START +{% if os == "windows" %} Using the **Command Prompt** (cmd), run the following: @@ -70,9 +70,9 @@ curl -L -o "%USERPROFILE%\wagon-de-bootcamp\terraform.tfvars" https://raw.github curl -L -o "%USERPROFILE%\wagon-de-bootcamp\.terraform.lock.hcl" https://raw.githubusercontent.com/lewagon/data-engineering-setup/main-automation/automation/infra/.terraform.lock.hcl ``` -$WINDOWS_END +{% endif %} -$LINUX_START +{% if os == "linux" %} ```bash mkdir -p ~/code/wagon-de-bootcamp @@ -85,27 +85,27 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le && curl -L -o ~/code/wagon-de-bootcamp/.terraform.lock.hcl https://raw.githubusercontent.com/lewagon/data-engineering-setup/main-automation/automation/infra/.terraform.lock.hcl ``` -$LINUX_END +{% endif %} ### Set variables -$MAC_START +{% if os == "macos" %} Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. -$MAC_END +{% endif %} -$WINDOWS_START +{% if os == "windows" %} Open up the file `C:\Users\\wagon-de-bootcamp\terraform.tfvars` in VS Code or any other code editor. -$WINDOWS_END +{% endif %} -$LINUX_START +{% if os == "linux" %} Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. -$LINUX_END +{% endif %} It should look like: @@ -122,15 +122,15 @@ We'll need to change some values in this file. Here's were you can find the requ - **region:** take a look at the GCP Region and Zone documentation at this [link here](https://cloud.google.com/compute/docs/regions-zones#available). We generally recommend you choose a geographically nearby region. - **zone:** Zone is a subset of region. it is almost always the same as **region** appended with `-a`, `-b`, or `-c`. The zone you select within a region should not have a functional impact. - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. -$MAC_START +{% if os == "macos" %} - **instance_user:** in your terminal, run `whoami`, and enter the value -$MAC_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} - **instance_user:** in Command Prompt, run `echo %username%`, and enter the value - try and remember your username, you will need it later on -$WINDOWS_END -$LINUX_START +{% endif %} +{% if os == "linux" %} - **instance_user:** in your terminal, run `whoami`, and enter the value -$LINUX_END +{% endif %} After completing this file, it might look similar to: @@ -145,15 +145,15 @@ instance_user = "taylorswift" Make sure to save the `terraform.tfvars` file, navigate into the directory with the terraform files using your terminal with: ```bash -$MAC_START +{% if os == "macos" %} cd ~/code/wagon-de-bootcamp -$MAC_END -$WINDOWS_START +{% endif %} +{% if os == "windows" %} cd %USERPROFILE%\wagon-de-bootcamp -$WINDOWS_END -$LINUX_START +{% endif %} +{% if os == "linux" %} cd ~/code/wagon-de-bootcamp -$LINUX_END +{% endif %} ``` Initialise and test the terraform config files with: diff --git a/_partials/vscode_ssh_connection.md b/_partials/vscode_ssh_connection.md index 55fcb58..c9f577f 100644 --- a/_partials/vscode_ssh_connection.md +++ b/_partials/vscode_ssh_connection.md @@ -20,7 +20,7 @@ For example, try running: # $ ssh lw-de-vm-.. ``` -$WINDOWS_START +{% if os == "windows" %} ### Confirm Your SSH Settings Let's take a look at the SSH configuration that was just created and verify it. In VS Code: @@ -92,7 +92,7 @@ icacls %USERPROFILE%\.ssh\google_compute_engine /grant:r %USERNAME%:(R) && ^ icacls %USERPROFILE%\.ssh\google_compute_engine /grant:r SYSTEM:(R) && ^ icacls %USERPROFILE%\.ssh\google_compute_engine ``` -$WINDOWS_END +{% endif %} ### Connect with VS Code diff --git a/build.rb b/build.rb index 54e576c..702f5a5 100755 --- a/build.rb +++ b/build.rb @@ -1,177 +1,22 @@ -#!/usr/bin/env ruby -wU -CONSTANTS = { - 'PYTHON_VERSION' => '3.8.12', - # 'PYTHON_CHECKER_URL' => 'https://raw.githubusercontent.com/lewagon/data-setup/master/checks/python_checker.sh', - # 'PIP_CHECKER_URL' => 'https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.sh', - # 'PIP_LOADER_URL' => 'https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.py', - 'CODE_EDITOR' => 'VS Code', - 'CODE_EDITOR_CMD' => 'code' -} - -# NOTE(ssaunier): This script needs https://github.com/lewagon/setup to be cloned as well -MAC_OS = %w[ - intro - setup/macos_slack - setup/slack_settings - setup/github - gcp_setup - osx_vscode - vscode_remote_ssh - gcp_cli_setup - gcp_cli_oauth - gcp_adc_auth - terraform - terraform_vm - vscode_ssh_connection - gcp_auth_vm_heading - gcp_cli_oauth - gcp_adc_auth - ubuntu_ansible_part1 - setup/gh_cli - dotfiles_simple - dotfiles_terminal - ubuntu_ansible_part2 - ubuntu_vm_test - repo_overview - dbeaver - setup/kitt -].freeze - -MAC_OS_KC = %w[ - keep_current - python_checkup -].freeze - -WINDOWS = %w[ - intro - setup/windows_slack - setup/slack_settings - setup/github - gcp_setup - win_vscode - vscode_remote_ssh - gcp_cli_setup - gcp_adc_auth - terraform - terraform_vm - vscode_ssh_connection - gcp_auth_vm_heading - gcp_cli_oauth - gcp_adc_auth - ubuntu_ansible_part1 - setup/gh_cli - dotfiles_simple - dotfiles_terminal - ubuntu_ansible_part2 - ubuntu_vm_test - repo_overview - dbeaver - setup/kitt -].freeze +#!/usr/bin/env ruby +# frozen_string_literal: true -WINDOWS_KC = %w[ - keep_current - python_checkup -].freeze +require 'yaml' +require_relative 'lib/builder' -LINUX = %w[ - intro - setup/ubuntu_slack - setup/slack_settings - setup/github - gcp_setup - setup/ubuntu_vscode - vscode_remote_ssh - gcp_cli_setup - gcp_cli_oauth - gcp_adc_auth - terraform - terraform_vm - vscode_ssh_connection - gcp_auth_vm_heading - gcp_cli_oauth - gcp_adc_auth - ubuntu_ansible_part1 - setup/gh_cli - dotfiles_simple - dotfiles_terminal - ubuntu_ansible_part2 - ubuntu_vm_test - repo_overview - dbeaver - setup/kitt -] +build_specs = Dir['builds/*.yml'].map { |filename| -LINUX_KC = %w[ - keep_current - python_checkup -] + data = YAML.load_file(filename) -LOCALES = [""] # english - -FILENAMES = { - "WINDOWS" => ["WINDOWS", WINDOWS], - "macOS" => ["macOS", MAC_OS], - "LINUX" => ["LINUX", LINUX], - "WINDOWS_keep_current" => ["WINDOWS", WINDOWS_KC], - "macOS_keep_current" => ["macOS", MAC_OS_KC], - "LINUX_keep_current" => ["LINUX", LINUX_KC] -} - -DELIMITERS = { - "WINDOWS" => ["\\$WINDOWS_START\n", "\\$WINDOWS_END\n"], - "macOS" => ["\\$MAC_START\n", "\\$MAC_END\n"], - "LINUX" => ["\\$LINUX_START\n", "\\$LINUX_END\n"] + BuildSpec.new( + name: File.basename(filename, '.yml'), + os: data['os'], + locales: data['locales'], + partials: data['partials'] + ) } -def load_partial(partial, locale) - match_data = partial.match(/setup\/(?[0-9a-z_]+)/) - partial = match_data[:partial] if match_data - partial = File.join(locale, partial) unless locale.empty? - file = File.join("_partials", "#{partial}.md") - if match_data - require 'open-uri' - content = URI.open(File.join("https://raw.githubusercontent.com/lewagon/setup/master", file)) - .string - # replace data-setup repo relative path by setup repo URL - image_paths = content.scan(/\!\[.*\]\((.*)\)/).flatten - image_paths.each { |ip| content.gsub!(ip, "https://github.com/lewagon/setup/blob/master/#{ip}")} - else - content = File.read(file, encoding: "utf-8") - end - return content -end - -# load partials -loaded = FILENAMES.map { |filename, (os_name, partials)| partials }.flatten.uniq -loaded = loaded.map { |partial| LOCALES.map { |locale| [partial, locale]} }.flatten(1) -loaded = loaded.map { |partial, locale| ["#{partial}.#{locale}", load_partial(partial, locale)] }.to_h +constants = YAML.load_file('constants/constants.yml').freeze +repos_cfg = YAML.load_file('constants/repos.yml').freeze -# write files -LOCALES.each do |locale| - FILENAMES.each do |filename, (os_name, partials)| - filename += ".#{locale}" unless locale.empty? - filename += ".md" - File.open(filename, "w:utf-8") do |f| - partials.each do |partial| - content = loaded["#{partial}.#{locale}"].clone - # remove the OS dependant blocks - removed_blocks = DELIMITERS.keys - [os_name] - removed_blocks.each do |block| - delimiter_start, delimiter_end = DELIMITERS[block] - pattern = "#{delimiter_start}(.|\n)*?(?", value) - end - f << content - f << "\n\n" - end - end - end -end +Builder.new(build_specs, constants, repos_cfg).run diff --git a/builds/LINUX.yml b/builds/LINUX.yml new file mode 100644 index 0000000..f06b274 --- /dev/null +++ b/builds/LINUX.yml @@ -0,0 +1,29 @@ + +os: linux +locales: [en] +partials: + - intro + - setup/ubuntu_slack + - setup/slack_settings + - setup/github + - gcp_setup + - setup/ubuntu_vscode + - vscode_remote_ssh + - gcp_cli_setup + - gcp_cli_oauth + - gcp_adc_auth + - terraform + - terraform_vm + - vscode_ssh_connection + - gcp_auth_vm_heading + - gcp_cli_oauth + - gcp_adc_auth + - ubuntu_ansible_part1 + - setup/gh_cli + - dotfiles_simple + - dotfiles_terminal + - ubuntu_ansible_part2 + - ubuntu_vm_test + - repo_overview + - dbeaver + - setup/kitt diff --git a/builds/LINUX_keep_current.yml b/builds/LINUX_keep_current.yml new file mode 100644 index 0000000..f69e9d9 --- /dev/null +++ b/builds/LINUX_keep_current.yml @@ -0,0 +1,6 @@ + +os: linux +locales: [en] +partials: + - keep_current + - python_checkup diff --git a/builds/WINDOWS.yml b/builds/WINDOWS.yml new file mode 100644 index 0000000..1ea1bd5 --- /dev/null +++ b/builds/WINDOWS.yml @@ -0,0 +1,28 @@ + +os: windows +locales: [en] +partials: + - intro + - setup/windows_slack + - setup/slack_settings + - setup/github + - gcp_setup + - win_vscode + - vscode_remote_ssh + - gcp_cli_setup + - gcp_adc_auth + - terraform + - terraform_vm + - vscode_ssh_connection + - gcp_auth_vm_heading + - gcp_cli_oauth + - gcp_adc_auth + - ubuntu_ansible_part1 + - setup/gh_cli + - dotfiles_simple + - dotfiles_terminal + - ubuntu_ansible_part2 + - ubuntu_vm_test + - repo_overview + - dbeaver + - setup/kitt diff --git a/builds/WINDOWS_keep_current.yml b/builds/WINDOWS_keep_current.yml new file mode 100644 index 0000000..e230908 --- /dev/null +++ b/builds/WINDOWS_keep_current.yml @@ -0,0 +1,6 @@ + +os: windows +locales: [en] +partials: + - keep_current + - python_checkup diff --git a/builds/macOS.yml b/builds/macOS.yml new file mode 100644 index 0000000..7bd8add --- /dev/null +++ b/builds/macOS.yml @@ -0,0 +1,29 @@ + +os: macos +locales: [en] +partials: + - intro + - setup/macos_slack + - setup/slack_settings + - setup/github + - gcp_setup + - osx_vscode + - vscode_remote_ssh + - gcp_cli_setup + - gcp_cli_oauth + - gcp_adc_auth + - terraform + - terraform_vm + - vscode_ssh_connection + - gcp_auth_vm_heading + - gcp_cli_oauth + - gcp_adc_auth + - ubuntu_ansible_part1 + - setup/gh_cli + - dotfiles_simple + - dotfiles_terminal + - ubuntu_ansible_part2 + - ubuntu_vm_test + - repo_overview + - dbeaver + - setup/kitt diff --git a/builds/macOS_keep_current.yml b/builds/macOS_keep_current.yml new file mode 100644 index 0000000..7665829 --- /dev/null +++ b/builds/macOS_keep_current.yml @@ -0,0 +1,6 @@ + +os: macos +locales: [en] +partials: + - keep_current + - python_checkup diff --git a/constants/constants.yml b/constants/constants.yml new file mode 100644 index 0000000..0dc6d8b --- /dev/null +++ b/constants/constants.yml @@ -0,0 +1,7 @@ + +PYTHON_VERSION: "3.8.12" +PYTHON_CHECKER_URL: "https://raw.githubusercontent.com/lewagon/data-setup/master/checks/python_checker.sh" +PIP_CHECKER_URL: "https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.sh" +PIP_LOADER_URL: "https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.py" +CODE_EDITOR: "VS Code" +CODE_EDITOR_CMD: "code" diff --git a/constants/repos.yml b/constants/repos.yml new file mode 100644 index 0000000..2fc6f3d --- /dev/null +++ b/constants/repos.yml @@ -0,0 +1,6 @@ + +branches: + setup: master + +aliases: + de_setup: data-engineering-setup diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..b0431d0 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,26 @@ + +# NOTE(ssaunier): This script needs https://github.com/lewagon/setup to be cloned as well + +## Build guides + +`bundle exec ruby build.rb` generates the setup guide markdown files at the repo root (e.g. `macOS.md`, `macOS.es.md`). + +Each file in `builds/` defines one guide: + +```yaml +os: macos +locales: [en, es] +partials: + - intro # local: _partials/intro.md / _partials/es/intro.md + - setup/github # remote: lewagon/setup _partials/github.md + - de_setup/direnv # remote alias: lewagon/data-engineering-setup _partials/direnv.md +``` + +Partials without a `/` are loaded from `_partials/` (English) or `_partials/{locale}/` (other locales). +Partials with a `/` are fetched from GitHub: `{repo}/_partials/{locale}/{name}.md`. Remote repo branches and aliases are configured in `constants/repos.yml`. + +Template variables available in all partials are defined in `constants/constants.yml`. + +CI runs the build on every push and commits the generated files if they changed. + +## Guide descriptions diff --git a/lib/build_spec.rb b/lib/build_spec.rb new file mode 100644 index 0000000..a86d164 --- /dev/null +++ b/lib/build_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require_relative 'localized_build' + +class BuildSpec + attr_reader :name, :os, :locales, :partials + + def initialize(name:, os:, locales:, partials:) + @name = name + @os = os + @locales = locales + @partials = partials + end + + def localized_builds(repos_cfg) + @locales.map { |locale| LocalizedBuild.new(self, locale, repos_cfg) } + end +end diff --git a/lib/builder.rb b/lib/builder.rb new file mode 100644 index 0000000..9a22437 --- /dev/null +++ b/lib/builder.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative 'build_spec' +require_relative 'partial_cache' +require_relative 'generator' + +class Builder + def initialize(build_specs, constants, repos_cfg) + @build_specs = build_specs + @constants = constants + @repos_cfg = repos_cfg + end + + def run + builds = @build_specs.flat_map { |spec| spec.localized_builds(@repos_cfg) } + cache = PartialCache.new(builds) + Generator.new(builds, cache, @constants).run + end +end diff --git a/lib/generator.rb b/lib/generator.rb new file mode 100644 index 0000000..55cfdca --- /dev/null +++ b/lib/generator.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'liquid' + +class Generator + def initialize(localized_builds, cache, constants) + @localized_builds = localized_builds + @cache = cache + @constants = constants + end + + def run + @localized_builds.each do |build| + File.open(build.output_filename, 'w:utf-8') do |f| + build.partials.each { |partial| f << render(partial, build) } + end + warn "built #{build.output_filename}" + end + end + + private + + def render(partial, build) + variables = @constants.merge(partial.vars).merge('os' => build.os, 'build_md' => build.output_filename) + "#{Liquid::Template.parse(@cache[partial.url].clone).render(variables)}\n\n" + end +end diff --git a/lib/localized_build.rb b/lib/localized_build.rb new file mode 100644 index 0000000..7bb8f9d --- /dev/null +++ b/lib/localized_build.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require_relative 'partial' + +class LocalizedBuild + attr_reader :os, :locale, :partials + + def initialize(spec, locale, repos_cfg) + @name = spec.name + @os = spec.os + @locale = locale + @partials = spec.partials.map { |entry| Partial.from(entry, locale, repos_cfg) } + end + + def output_filename + @locale == 'en' ? "#{@name}.md" : "#{@name}.#{@locale}.md" + end +end diff --git a/lib/partial.rb b/lib/partial.rb new file mode 100644 index 0000000..a7f3951 --- /dev/null +++ b/lib/partial.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative 'partial/remote' +require_relative 'partial/local' + +module Partial + REMOTE_REGEX = %r{\A(?[a-z][a-z0-9_-]*)/(?[a-z0-9_]+)\z} + + # entry is either a plain string or a hash: { 'name' => '...', 'vars' => { ... } } + def self.from(entry, locale, repos_cfg) + name = entry.is_a?(Hash) ? entry['name'] : entry + vars = entry.is_a?(Hash) ? entry.fetch('vars', {}) : {} + if (m = name.match(REMOTE_REGEX)) + Remote.new(m[:repo], locale, m[:name], vars, repos_cfg) + else + Local.new(locale, name, vars) + end + end +end diff --git a/lib/partial/local.rb b/lib/partial/local.rb new file mode 100644 index 0000000..92bb992 --- /dev/null +++ b/lib/partial/local.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Partial + class Local + attr_reader :url, :vars + + def initialize(locale, name, vars) + localized = "_partials/#{locale}/#{name}.md" + english = "_partials/#{name}.md" + @url = (locale != 'en' && File.exist?(localized)) ? localized : english + @vars = vars + end + + def content = File.read(@url, encoding: 'utf-8') + end +end diff --git a/lib/partial/remote.rb b/lib/partial/remote.rb new file mode 100644 index 0000000..33a86ba --- /dev/null +++ b/lib/partial/remote.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'open-uri' + +module Partial + class Remote + attr_reader :url, :vars + + def initialize(repo, locale, name, vars, repos_cfg) + repo = repos_cfg.dig('aliases', repo) || repo + branch = repos_cfg.dig('branches', repo) || 'main' + path = locale == 'en' ? name : "#{locale}/#{name}" + @base_url = "https://github.com/lewagon/#{repo}/blob/#{branch}" + @url = "https://raw.githubusercontent.com/lewagon/#{repo}/#{branch}/_partials/#{path}.md" + @vars = vars + end + + def content + text = URI.open(@url).read + # Rewrite relative image paths to absolute GitHub URLs so they render outside their source repo + text.scan(/\!\[.*\]\((.*)\)/).flatten + .reject { |ip| ip.start_with?('http') } + .each { |ip| text.gsub!(ip, "#{@base_url}/#{ip}") } + text.scan(/src="(images\/.*)"/).flatten + .each { |ip| text.gsub!(ip, "#{@base_url}/#{ip}") } + warn "fetched #{@url}" + text + end + end +end diff --git a/lib/partial_cache.rb b/lib/partial_cache.rb new file mode 100644 index 0000000..91ab0c1 --- /dev/null +++ b/lib/partial_cache.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class PartialCache + def initialize(localized_builds) + @cache = localized_builds + .flat_map(&:partials) + .uniq { |partial| partial.url } + .sort_by { |partial| partial.url } + .map { |partial| Thread.new { [partial.url, partial.content] } } + .map { |thread| thread.value } + .to_h + end + + def [](url) + @cache[url] + end +end diff --git a/macOS.md b/macOS.md index 45483e3..e989bb0 100644 --- a/macOS.md +++ b/macOS.md @@ -289,6 +289,7 @@ The `gcloud` Command Line Interface (CLI) is used to communicate with Google Clo ### Install gcloud + #### 1. Check to see if python 3 is available, otherwise install If you don't know, run: @@ -368,6 +369,9 @@ gcloud --version + + + ### Authenticate gcloud We need to authenticate the `gcloud` CLI tool and set the project so it can interact with Google from the terminal. @@ -390,6 +394,8 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog + + You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: @@ -437,6 +443,7 @@ And follow the prompts. It should open a web-page to login to your Google accoun Terraform is a tool for [Infrastructure as Code (IaC) 🔗](https://en.wikipedia.org/wiki/Infrastructure_as_code) to create, destroy, and manage resources in the cloud. + To install Terraform, navigate to the [Terraform downloads page 🔗](https://developer.hashicorp.com/terraform/install): - Under **Binary download**, select the binary that corresponds to your CPU architecture: - M chip (arm64): ARM64 @@ -451,6 +458,9 @@ To install Terraform, navigate to the [Terraform downloads page 🔗](https://de + + + Verify the installation with: ```bash @@ -463,9 +473,13 @@ The output should look similar to: Terraform v1.14.3 on _ + # macOS (M chip) example # Terraform 1.14.3 # on darwin_arm64 + + + ``` @@ -508,6 +522,7 @@ We almost have all the necessary parts to create your VM using **terraform**. We First we'll create a folder and download the terraform files with: + In a terminal, run the following commands: ```bash @@ -524,14 +539,21 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le + + + ### Set variables + Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. + + + It should look like: ```bash @@ -547,8 +569,12 @@ We'll need to change some values in this file. Here's were you can find the requ - **region:** take a look at the GCP Region and Zone documentation at this [link here](https://cloud.google.com/compute/docs/regions-zones#available). We generally recommend you choose a geographically nearby region. - **zone:** Zone is a subset of region. it is almost always the same as **region** appended with `-a`, `-b`, or `-c`. The zone you select within a region should not have a functional impact. - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. + - **instance_user:** in your terminal, run `whoami`, and enter the value + + + After completing this file, it might look similar to: ```bash @@ -562,7 +588,11 @@ instance_user = "taylorswift" Make sure to save the `terraform.tfvars` file, navigate into the directory with the terraform files using your terminal with: ```bash + cd ~/code/wagon-de-bootcamp + + + ``` Initialise and test the terraform config files with: @@ -627,6 +657,7 @@ For example, try running: ``` + ### Connect with VS Code To connect to your Virtual Machine, click on the small symbol at the very bottom-left corner of VS Code: @@ -701,6 +732,8 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog + + You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: @@ -823,7 +856,7 @@ We will use the GitHub CLI (`gh`) to connect to GitHub using *SSH*, a protocol t First in order to **login**, copy-paste the following command in your terminal: -:warning: **DO NOT edit the `email`** +:warning: **DO NOT edit the `email`** — Even though `user:email` looks like a placeholder for your actual email address, it isn't — do not replace it. ```bash gh auth login -s 'user:email' -w --git-protocol ssh @@ -835,7 +868,9 @@ gh auth login -s 'user:email' -w --git-protocol ssh If you already have SSH keys, you will see instead `Upload your SSH public key to your GitHub account?` With the arrows, select your public key file path and press `Enter`. -- `Enter a passphrase for your new SSH key (Optional)`. Type something you want and that you'll remember. It's a password to protect your private key stored on your hard drive. Then press `Enter`. +- `Enter a passphrase for your new SSH key (Optional)`: + - **FOR MOST PEOPLE:** Just press `Enter` to skip. You don't need a passphrase for the bootcamp and it would prompt you every time you use the key. There is a risk, however, that if someone steals your laptop, they could then push to GitHub. + - **IF SECURITY IS REALLY IMPORTANT TO YOU:** Enter a passphrase of your choice and press `Enter`. It's _really_ important that if you enter a passphrase, you write it down somewhere immediately and do not lose/forget it. You will need to enter this frequently. - `Title for your SSH key`. You can leave it at the proposed "GitHub CLI", press `Enter`. diff --git a/macOS_keep_current.md b/macOS_keep_current.md index 2ed13b1..bf25864 100644 --- a/macOS_keep_current.md +++ b/macOS_keep_current.md @@ -86,10 +86,14 @@ type -a pyenv > /dev/null && eval "$(pyenv init --path)" Update pyenv : + ``` bash brew update && brew upgrade pyenv ``` + + + Install the current python version : ```bash @@ -140,6 +144,7 @@ pyenv versions pip install -U pip ``` + If your computer uses **Apple Silicon**, expand the paragraph below and go through it. Otherwise ignore it.
@@ -160,6 +165,9 @@ pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs ```
+ + + ## GCP Make sure that the `gcloud` command is linked to the email address of your Google Cloud Platform account : @@ -228,8 +236,7 @@ cat $GOOGLE_APPLICATION_CREDENTIALS { "type": "service_account", "project_id": "your-gcp-project-id", - "private_key_id": "a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4a2d4", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInMIInM=\n-----END PRIVATE KEY-----\n", + "private_key_id": "...", "client_email": "your-service-account@your-service-account.iam.gserviceaccount.com", "client_id": "105410541054105410541", "auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -268,8 +275,12 @@ gcloud auth configure-docker ## Docker + Start the Docker app + + + Verify that Docker can run the hello-world image : ``` bash @@ -278,24 +289,28 @@ docker run hello-world 👉 Make sure that this command completes correctly + Stop the Docker app + + + ### Python setup check up Check your Python version with the following commands: ```bash -zsh -c "$(curl -fsSL )" 3.8.12 +zsh -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/python_checker.sh)" 3.8.12 ``` Run the following command to check if you successfully installed the required packages: ```bash -zsh -c "$(curl -fsSL )" +zsh -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.sh)" ``` Now run the following command to check if you can load these packages: ```bash -python -c "$(curl -fsSL )" +python -c "$(curl -fsSL https://raw.githubusercontent.com/lewagon/data-setup/master/checks/pip_check.py)" ``` Make sure you can run Jupyter: From cd8fa50e749a227b0797a26cfbe0cf528385ca5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Manchon?= Date: Wed, 3 Jun 2026 11:15:12 +0200 Subject: [PATCH 2/6] style --- LINUX.md | 24 ------------------------ LINUX_keep_current.md | 8 -------- WINDOWS.md | 22 ---------------------- WINDOWS_keep_current.md | 8 -------- _partials/gcp_cli_oauth.md | 4 +--- _partials/gcp_cli_setup.md | 8 ++------ _partials/keep_current.md | 24 ++++++++---------------- _partials/terraform.md | 8 ++------ _partials/terraform_vm.md | 28 ++++++++-------------------- macOS.md | 24 ------------------------ macOS_keep_current.md | 8 -------- 11 files changed, 21 insertions(+), 145 deletions(-) diff --git a/LINUX.md b/LINUX.md index 81a6f48..97773c9 100644 --- a/LINUX.md +++ b/LINUX.md @@ -325,10 +325,6 @@ The `gcloud` Command Line Interface (CLI) is used to communicate with Google Clo - - - - Add the `APT` repository and install with: ```bash @@ -371,8 +367,6 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog - - For pasting into the terminal, your might need to use `ctrl + shift + v` @@ -425,10 +419,6 @@ Terraform is a tool for [Infrastructure as Code (IaC) 🔗](https://en.wikipedia - - - - Install some system requirements requirements: ```bash sudo apt-get update && sudo apt-get install -y gnupg software-properties-common @@ -523,10 +513,6 @@ First we'll create a folder and download the terraform files with: - - - - ```bash mkdir -p ~/code/wagon-de-bootcamp ``` @@ -544,10 +530,6 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le - - - - Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. @@ -568,8 +550,6 @@ We'll need to change some values in this file. Here's were you can find the requ - **zone:** Zone is a subset of region. it is almost always the same as **region** appended with `-a`, `-b`, or `-c`. The zone you select within a region should not have a functional impact. - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. - - - **instance_user:** in your terminal, run `whoami`, and enter the value @@ -587,8 +567,6 @@ Make sure to save the `terraform.tfvars` file, navigate into the directory with ```bash - - cd ~/code/wagon-de-bootcamp ``` @@ -730,8 +708,6 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog - - For pasting into the terminal, your might need to use `ctrl + shift + v` diff --git a/LINUX_keep_current.md b/LINUX_keep_current.md index 56a19c6..8131cac 100644 --- a/LINUX_keep_current.md +++ b/LINUX_keep_current.md @@ -87,13 +87,11 @@ type -a pyenv > /dev/null && eval "$(pyenv init --path)" Update pyenv : - ``` bash cd $(pyenv root) && git pull ``` - Install the current python version : ```bash @@ -145,8 +143,6 @@ pip install -U pip ``` - - ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` @@ -260,7 +256,6 @@ gcloud auth configure-docker ## Docker - Start Docker : ``` bash @@ -268,7 +263,6 @@ sudo service docker start ``` - Verify that Docker can run the hello-world image : ``` bash @@ -278,7 +272,6 @@ docker run hello-world 👉 Make sure that this command completes correctly - Start Docker : ``` bash @@ -287,7 +280,6 @@ sudo service docker stop - ### Python setup check up Check your Python version with the following commands: diff --git a/WINDOWS.md b/WINDOWS.md index c4bf7cc..14531c7 100644 --- a/WINDOWS.md +++ b/WINDOWS.md @@ -292,8 +292,6 @@ The `gcloud` Command Line Interface (CLI) is used to communicate with Google Clo - - To install, download the Google Cloud CLI installer from this [link here 🔗](https://cloud.google.com/sdk/docs/install#windows). Once it's finished downloading, launch the installer and follow the prompts. You only need to install `gcloud` for the current user. @@ -328,8 +326,6 @@ Now `gcloud` is installed and authenticated 🚀 - - ### Application Default Credentials Application Default Credentials are for authenticating our **code** (Terraform and Python 🐍) to interact with Google services and resources. It's a small distinction between `gcloud` and **code**, but an important one. @@ -349,8 +345,6 @@ Terraform is a tool for [Infrastructure as Code (IaC) 🔗](https://en.wikipedia - - ### Download To install terraform, download the **zip archive** from the Terraform install page at this [link here 🔗](https://developer.hashicorp.com/terraform/install). @@ -391,8 +385,6 @@ To update your path: - - Verify the installation with: ```bash @@ -455,8 +447,6 @@ First we'll create a folder and download the terraform files with: - - Using the **Command Prompt** (cmd), run the following: ❗ Note: The below commands will only work in **Command Prompt** - they will not work in Windows Powershell. @@ -474,20 +464,14 @@ curl -L -o "%USERPROFILE%\wagon-de-bootcamp\.terraform.lock.hcl" https://raw.git - - ### Set variables - - Open up the file `C:\Users\\wagon-de-bootcamp\terraform.tfvars` in VS Code or any other code editor. - - It should look like: ```bash @@ -504,11 +488,9 @@ We'll need to change some values in this file. Here's were you can find the requ - **zone:** Zone is a subset of region. it is almost always the same as **region** appended with `-a`, `-b`, or `-c`. The zone you select within a region should not have a functional impact. - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. - - **instance_user:** in Command Prompt, run `echo %username%`, and enter the value - try and remember your username, you will need it later on - After completing this file, it might look similar to: ```bash @@ -523,10 +505,8 @@ Make sure to save the `terraform.tfvars` file, navigate into the directory with ```bash - cd %USERPROFILE%\wagon-de-bootcamp - ``` Initialise and test the terraform config files with: @@ -742,8 +722,6 @@ For pasting into the terminal, your might need to use `ctrl + shift + v` - - You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: diff --git a/WINDOWS_keep_current.md b/WINDOWS_keep_current.md index f139e8c..aa7acbf 100644 --- a/WINDOWS_keep_current.md +++ b/WINDOWS_keep_current.md @@ -87,8 +87,6 @@ type -a pyenv > /dev/null && eval "$(pyenv init --path)" Update pyenv : - - ``` bash cd $(pyenv root) && git pull ``` @@ -145,13 +143,11 @@ pip install -U pip ``` - ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` - ## GCP Make sure that the `gcloud` command is linked to the email address of your Google Cloud Platform account : @@ -260,8 +256,6 @@ gcloud auth configure-docker ## Docker - - Start Docker : ``` bash @@ -278,8 +272,6 @@ docker run hello-world 👉 Make sure that this command completes correctly - - Start Docker : ``` bash diff --git a/_partials/gcp_cli_oauth.md b/_partials/gcp_cli_oauth.md index 1f6b972..9164cf7 100644 --- a/_partials/gcp_cli_oauth.md +++ b/_partials/gcp_cli_oauth.md @@ -22,9 +22,7 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog For pasting into the terminal, your might need to use `ctrl + shift + v` -{% endif %} - -{% if os == "linux" %} +{% elsif os == "linux" %} For pasting into the terminal, your might need to use `ctrl + shift + v` diff --git a/_partials/gcp_cli_setup.md b/_partials/gcp_cli_setup.md index 3040710..001d6a7 100644 --- a/_partials/gcp_cli_setup.md +++ b/_partials/gcp_cli_setup.md @@ -81,9 +81,7 @@ Once the installer is finished, test the install by running: gcloud --version ``` -{% endif %} - -{% if os == "windows" %} +{% elsif os == "windows" %} To install, download the Google Cloud CLI installer from this [link here 🔗](https://cloud.google.com/sdk/docs/install#windows). @@ -116,9 +114,7 @@ Your active configurations: [default] Now `gcloud` is installed and authenticated 🚀 -{% endif %} - -{% if os == "linux" %} +{% elsif os == "linux" %} Add the `APT` repository and install with: diff --git a/_partials/keep_current.md b/_partials/keep_current.md index 93e13d5..7076250 100644 --- a/_partials/keep_current.md +++ b/_partials/keep_current.md @@ -90,13 +90,11 @@ Update pyenv : ``` bash brew update && brew upgrade pyenv ``` -{% endif %} -{% if os == "linux" %} +{% elsif os == "linux" %} ``` bash cd $(pyenv root) && git pull ``` -{% endif %} -{% if os == "windows" %} +{% elsif os == "windows" %} ``` bash cd $(pyenv root) && git pull ``` @@ -172,13 +170,11 @@ If your computer uses **Apple Intel**, expand the paragraph below and go through pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/apple_intel.txt ``` -{% endif %} -{% if os == "windows" %} +{% elsif os == "windows" %} ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` -{% endif %} -{% if os == "linux" %} +{% elsif os == "linux" %} ``` bash pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs/releases/linux.txt ``` @@ -293,15 +289,13 @@ gcloud auth configure-docker {% if os == "macos" %} Start the Docker app -{% endif %} -{% if os == "linux" %} +{% elsif os == "linux" %} Start Docker : ``` bash sudo service docker start ``` -{% endif %} -{% if os == "windows" %} +{% elsif os == "windows" %} Start Docker : ``` bash @@ -319,15 +313,13 @@ docker run hello-world {% if os == "macos" %} Stop the Docker app -{% endif %} -{% if os == "linux" %} +{% elsif os == "linux" %} Start Docker : ``` bash sudo service docker stop ``` -{% endif %} -{% if os == "windows" %} +{% elsif os == "windows" %} Start Docker : ``` bash diff --git a/_partials/terraform.md b/_partials/terraform.md index 6b7bc81..be653df 100644 --- a/_partials/terraform.md +++ b/_partials/terraform.md @@ -15,9 +15,7 @@ To install Terraform, navigate to the [Terraform downloads page 🔗](https://de sudo cp ~/Downloads/terraform_*/terraform /usr/local/bin/ ``` -{% endif %} - -{% if os == "windows" %} +{% elsif os == "windows" %} ### Download @@ -57,9 +55,7 @@ To update your path: 6. Close **Command Prompt** and open it again -{% endif %} - -{% if os == "linux" %} +{% elsif os == "linux" %} Install some system requirements requirements: ```bash diff --git a/_partials/terraform_vm.md b/_partials/terraform_vm.md index 76a8523..cf269e7 100644 --- a/_partials/terraform_vm.md +++ b/_partials/terraform_vm.md @@ -51,9 +51,7 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le && curl -L -o ~/code/wagon-de-bootcamp/.terraform.lock.hcl https://raw.githubusercontent.com/lewagon/data-engineering-setup/main-automation/automation/infra/.terraform.lock.hcl ``` -{% endif %} - -{% if os == "windows" %} +{% elsif os == "windows" %} Using the **Command Prompt** (cmd), run the following: @@ -70,9 +68,7 @@ curl -L -o "%USERPROFILE%\wagon-de-bootcamp\terraform.tfvars" https://raw.github curl -L -o "%USERPROFILE%\wagon-de-bootcamp\.terraform.lock.hcl" https://raw.githubusercontent.com/lewagon/data-engineering-setup/main-automation/automation/infra/.terraform.lock.hcl ``` -{% endif %} - -{% if os == "linux" %} +{% elsif os == "linux" %} ```bash mkdir -p ~/code/wagon-de-bootcamp @@ -93,15 +89,11 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. -{% endif %} - -{% if os == "windows" %} +{% elsif os == "windows" %} Open up the file `C:\Users\\wagon-de-bootcamp\terraform.tfvars` in VS Code or any other code editor. -{% endif %} - -{% if os == "linux" %} +{% elsif os == "linux" %} Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any other code editor. @@ -124,11 +116,9 @@ We'll need to change some values in this file. Here's were you can find the requ - **instance_name:** we recommend naming your VM: `lw-de-vm-`. Replacing `` with your GitHub username. {% if os == "macos" %} - **instance_user:** in your terminal, run `whoami`, and enter the value -{% endif %} -{% if os == "windows" %} +{% elsif os == "windows" %} - **instance_user:** in Command Prompt, run `echo %username%`, and enter the value - try and remember your username, you will need it later on -{% endif %} -{% if os == "linux" %} +{% elsif os == "linux" %} - **instance_user:** in your terminal, run `whoami`, and enter the value {% endif %} @@ -147,11 +137,9 @@ Make sure to save the `terraform.tfvars` file, navigate into the directory with ```bash {% if os == "macos" %} cd ~/code/wagon-de-bootcamp -{% endif %} -{% if os == "windows" %} +{% elsif os == "windows" %} cd %USERPROFILE%\wagon-de-bootcamp -{% endif %} -{% if os == "linux" %} +{% elsif os == "linux" %} cd ~/code/wagon-de-bootcamp {% endif %} ``` diff --git a/macOS.md b/macOS.md index e989bb0..9eac5b8 100644 --- a/macOS.md +++ b/macOS.md @@ -368,10 +368,6 @@ gcloud --version - - - - ### Authenticate gcloud We need to authenticate the `gcloud` CLI tool and set the project so it can interact with Google from the terminal. @@ -394,8 +390,6 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog - - You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: @@ -457,10 +451,6 @@ To install Terraform, navigate to the [Terraform downloads page 🔗](https://de - - - - Verify the installation with: ```bash @@ -538,10 +528,6 @@ curl -L -o ~/code/wagon-de-bootcamp/main.tf https://raw.githubusercontent.com/le - - - - ### Set variables @@ -550,10 +536,6 @@ Open up the file `~/code/wagon-de-bootcamp/terraform.tfvars` in VS Code or any o - - - - It should look like: ```bash @@ -573,8 +555,6 @@ We'll need to change some values in this file. Here's were you can find the requ - **instance_user:** in your terminal, run `whoami`, and enter the value - - After completing this file, it might look similar to: ```bash @@ -591,8 +571,6 @@ Make sure to save the `terraform.tfvars` file, navigate into the directory with cd ~/code/wagon-de-bootcamp - - ``` Initialise and test the terraform config files with: @@ -732,8 +710,6 @@ We recommend allowing **Google Auth Library** to: _View and sign in to your Goog - - You also need to set the GCP project that your are working in. For this section, you'll need your **GCP Project ID**, which can be found on the GCP Console at this [link here 🔗](https://console.cloud.google.com). Makes sure you copy the _Project ID_ and **not** the _Project number_. To set your project, replace `` with your GCP Project ID and run: diff --git a/macOS_keep_current.md b/macOS_keep_current.md index bf25864..03d4596 100644 --- a/macOS_keep_current.md +++ b/macOS_keep_current.md @@ -92,8 +92,6 @@ brew update && brew upgrade pyenv ``` - - Install the current python version : ```bash @@ -166,8 +164,6 @@ pip install -r https://raw.githubusercontent.com/lewagon/data-setup/master/specs - - ## GCP Make sure that the `gcloud` command is linked to the email address of your Google Cloud Platform account : @@ -279,8 +275,6 @@ gcloud auth configure-docker Start the Docker app - - Verify that Docker can run the hello-world image : ``` bash @@ -294,8 +288,6 @@ Stop the Docker app - - ### Python setup check up Check your Python version with the following commands: From 5a8597a0ba73d6d3fe094058ccc7cda48d610645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Manchon?= Date: Wed, 3 Jun 2026 14:23:58 +0200 Subject: [PATCH 3/6] fix output order --- lib/partial/remote.rb | 1 - lib/partial_cache.rb | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/partial/remote.rb b/lib/partial/remote.rb index 33a86ba..01a0900 100644 --- a/lib/partial/remote.rb +++ b/lib/partial/remote.rb @@ -23,7 +23,6 @@ def content .each { |ip| text.gsub!(ip, "#{@base_url}/#{ip}") } text.scan(/src="(images\/.*)"/).flatten .each { |ip| text.gsub!(ip, "#{@base_url}/#{ip}") } - warn "fetched #{@url}" text end end diff --git a/lib/partial_cache.rb b/lib/partial_cache.rb index 91ab0c1..e425148 100644 --- a/lib/partial_cache.rb +++ b/lib/partial_cache.rb @@ -8,6 +8,7 @@ def initialize(localized_builds) .sort_by { |partial| partial.url } .map { |partial| Thread.new { [partial.url, partial.content] } } .map { |thread| thread.value } + .tap { |results| results.each { |url, _| warn "fetched #{url}" } } .to_h end From b09dd631fc50e21884ceaefa9bf770c81b5edf5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Manchon?= Date: Wed, 3 Jun 2026 16:04:20 +0200 Subject: [PATCH 4/6] ignore bundle build outputs --- .github/workflows/build.yml | 4 ++-- .gitignore | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6b731f..d6e3270 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,13 +4,13 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: bundler-cache: true - - name: Install dependencies - run: bundle install - name: Build run: | git config user.name github-actions diff --git a/.gitignore b/.gitignore index 3e9297d..2db2ebd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .DS_Store .ipynb_ckeckpoints +.bundle/ +vendor/ ## Terraform **/.terraform/* From 130f6ce24e7ff3a9469ac9963e40451db952225d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Manchon?= Date: Wed, 3 Jun 2026 16:07:31 +0200 Subject: [PATCH 5/6] test --- _partials/intro.md | 1 + 1 file changed, 1 insertion(+) diff --git a/_partials/intro.md b/_partials/intro.md index e5c2ba6..bc63baa 100644 --- a/_partials/intro.md +++ b/_partials/intro.md @@ -1,3 +1,4 @@ + # Setup instructions You will find below the instructions to set up your computer for [Le Wagon Data Engineering course](https://www.lewagon.com/) From 2c22f59664b2e6824f8a62afac1d7f8429a99c78 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 3 Jun 2026 14:07:49 +0000 Subject: [PATCH 6/6] setup guides generated --- LINUX.md | 1 + WINDOWS.md | 1 + macOS.md | 1 + 3 files changed, 3 insertions(+) diff --git a/LINUX.md b/LINUX.md index 97773c9..e7226ee 100644 --- a/LINUX.md +++ b/LINUX.md @@ -1,3 +1,4 @@ + # Setup instructions You will find below the instructions to set up your computer for [Le Wagon Data Engineering course](https://www.lewagon.com/) diff --git a/WINDOWS.md b/WINDOWS.md index 14531c7..9c7b223 100644 --- a/WINDOWS.md +++ b/WINDOWS.md @@ -1,3 +1,4 @@ + # Setup instructions You will find below the instructions to set up your computer for [Le Wagon Data Engineering course](https://www.lewagon.com/) diff --git a/macOS.md b/macOS.md index 9eac5b8..b35e9dc 100644 --- a/macOS.md +++ b/macOS.md @@ -1,3 +1,4 @@ + # Setup instructions You will find below the instructions to set up your computer for [Le Wagon Data Engineering course](https://www.lewagon.com/)