Skip to content

Commit

Permalink
Merge branch 'main' into feat/update-lassie-v0.21.0
Browse files Browse the repository at this point in the history
Signed-off-by: Diego Rodríguez Baquero <diego@protocol.ai>
  • Loading branch information
Diego Rodríguez Baquero committed Jan 17, 2024
2 parents 70066cb + e7be219 commit 569ae65
Show file tree
Hide file tree
Showing 15 changed files with 367 additions and 179 deletions.
30 changes: 19 additions & 11 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,42 +23,50 @@ jobs:
build-and-push-image:
runs-on: larger

strategy:
matrix:
platform: [linux/amd64, linux/arm64]

permissions:
contents: read
packages: write

steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@v1
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

- name: Set orchestrator env (main)
- name: Set Up env (main)
if: github.ref_type == 'tag'
run: echo "NETWORK=main" >> $GITHUB_ENV
run: |
echo "NETWORK=main" >> $GITHUB_ENV
printf ${{ secrets.PRODUCTION_BASE64_JWT_PUBLIC_KEY }} | base64 --decode > ./container/nginx/jwt_pub.key
- name: Set orchestrator env (test)
- name: Set Up env (test)
if: github.ref_type == 'branch'
run: echo "NETWORK=test" >> $GITHUB_ENV
run: |
echo "NETWORK=test" >> $GITHUB_ENV
printf ${{ secrets.STAGING_BASE64_JWT_PUBLIC_KEY }} | base64 --decode > ./container/nginx/jwt_pub.key
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
with:
platforms: "arm64"

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3

- name: Set the git short sha
id: git
Expand Down Expand Up @@ -91,15 +99,15 @@ jobs:
echo "version_docker_tags=$repo:${{ github.run_number}}_${{ steps.git.outputs.sha_short }}"
- name: Build and push Docker image
uses: docker/build-push-action@v3
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: |
${{ steps.tags.outputs.version_docker_tags }}
${{ steps.tags.outputs.mutable_docker_tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64
platforms: ${{ matrix.platform }}
cache-from: type=gha,scope=l1
cache-to: type=gha,mode=max,scope=l1
build-args: |
Expand Down
38 changes: 36 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ ARG NGX_BROTLI_COMMIT=6e975bcb015f62e1f303054897783355e2a877dc
ARG NODEJS_MAJOR_VERSION="18"
# https://github.com/filecoin-project/lassie/releases
ARG LASSIE_VERSION="v0.21.0"
# https://github.com/max-lt/nginx-jwt-module
ARG NGINX_JWT_VERSION="v3.2.2"
ARG LIBJWT_VERSION=1.15.3

#############
# nginx build
Expand All @@ -21,6 +24,8 @@ FROM docker.io/library/debian:bullseye AS build
ARG NGINX_VERSION
ARG NGX_BROTLI_COMMIT
ARG NJS_VERSION
ARG NGINX_JWT_VERSION
ARG LIBJWT_VERSION

# Install dependencies
RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests \
Expand All @@ -45,6 +50,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
clang \
&& rm -rf /var/lib/apt/lists/*


# Install jwt dependencies
RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests \
libjansson-dev \
autoconf \
automake \
libtool \
pkg-config \
check \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /usr/src

RUN echo "Cloning brotli $NGX_BROTLI_COMMIT" \
Expand All @@ -64,6 +80,20 @@ RUN echo "Cloning njs $NJS_VERSION" \
&& ./configure \
&& make

RUN echo "Cloning nginx-jwt-module $NGINX_JWT_VERSION" \
&& git clone --depth 1 --branch $NGINX_JWT_VERSION https://github.com/max-lt/nginx-jwt-module.git

RUN echo "Installing libjwt $LIBJWT_VERSION" \
&& mkdir libjwt \
&& curl -sL https://github.com/benmcollins/libjwt/archive/v${LIBJWT_VERSION}.tar.gz \
| tar -zx -C libjwt/ --strip-components=1 \
&& cd libjwt \
&& autoreconf -i \
&& ./configure \
&& make all \
&& make check \
&& make install

ARG CONFIG="--prefix=/etc/nginx \
--sbin-path=/usr/sbin/nginx \
--modules-path=/usr/lib/nginx/modules \
Expand All @@ -86,7 +116,8 @@ ARG CONFIG="--prefix=/etc/nginx \
--with-http_sub_module \
--with-http_v2_module \
--add-dynamic-module=/usr/src/ngx_brotli \
--add-dynamic-module=/usr/src/njs/nginx"
--add-dynamic-module=/usr/src/njs/nginx \
--add-dynamic-module=/usr/src/nginx-jwt-module"

RUN echo "Downloading and extracting nginx $NGINX_VERSION" \
&& mkdir /usr/src/nginx \
Expand All @@ -111,6 +142,9 @@ COPY --from=build /usr/sbin/nginx /usr/sbin/
COPY --from=build /usr/src/nginx/objs/ngx_http_brotli_filter_module.so /usr/lib/nginx/modules/
COPY --from=build /usr/src/nginx/objs/ngx_http_brotli_static_module.so /usr/lib/nginx/modules/
COPY --from=build /usr/src/nginx/objs/ngx_http_js_module.so /usr/lib/nginx/modules/
COPY --from=build /usr/lib/nginx/modules/ngx_http_auth_jwt_module.so /usr/lib/nginx/modules/
COPY --from=build /usr/local/lib/libjwt.so /lib


# Prepare
RUN apt-get update \
Expand All @@ -122,7 +156,7 @@ RUN apt-get update \

# Install dependencies
RUN apt-get update \
&& apt-get install --no-install-recommends -y nodejs speedtest logrotate jq \
&& apt-get install --no-install-recommends -y nodejs speedtest logrotate jq libjansson-dev \
&& rm -rf /var/lib/apt/lists/*

# Download lassie
Expand Down
24 changes: 17 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ on [Filecoin Slack](https://filecoinproject.slack.com/)! 👋
- [Switch networks between test net and main net](#switch-networks-between-test-net-and-main-net)
- [Node operator guide](#node-operator-guide)
- [Obtaining a Filecoin wallet address](#obtaining-a-filecoin-wallet-address)
- [Receiving FIL payments](#receiving-fil-payments)
- [Claiming your node's earnings](#claiming-your-earnings)
- [Node monitoring](#node-monitoring)
- [Community Tools](#community-tools)
- [License](#license)

## Requirements
Expand Down Expand Up @@ -229,7 +230,7 @@ If you want to switch your node from Saturn's test network (aka `test`) to Satur

## Node operator guide

For answers to common questions about operating a node, like about receiving your filecoin payouts, see the L1 node [FAQ](docs/faq.md) page.
For answers to common questions about operating a node, see the L1 node [FAQ](docs/faq.md) page.

### Network Uptime Requirement

Expand All @@ -252,7 +253,7 @@ Read more about Saturn's node uptime requirement in the docs, [here](https://doc
### Obtaining a Filecoin wallet address
You need to own a Filecoin wallet to receive FIL payments.
You need to own a Filecoin wallet to receive FIL payouts.
- [Official Filecoin wallet documentation](https://docs.filecoin.io/get-started/overview/#wallets)
Expand All @@ -273,14 +274,23 @@ The Saturn team will **never** DM you or ask you to verify/validate/upgrade your
please ask in public channels such as the [#filecoin-saturn](https://filecoinproject.slack.com/archives/C03DH0BL02E)
Slack.

### Receiving FIL payments
### Claiming your earnings

When payments are scheduled to be sent out, your Filecoin wallet will receive a FIL payment.
Each month, your node's earnings, in FIL, are calculated by the network based on various factors such as the amount of bandwidth it served, the number of requests it handled, its performance metrics like TTFB and upload speed, and its availaility and uptime. These earnings are then sent to a payout FVM smart contract by the 7th day of the following month. For example, earnings for December 2022 would be transferred to a payout smart contract by January 7th, 2023.
After your node's earnings are in the payout FVM smart contract, you can claim them on [payouts.saturn.tech](https://payouts.saturn.tech). Claiming your earnings moves the Filecoin your node(s) earned from the smart contract to your personal Filecoin wallet.

### Node monitoring

- https://dashboard.saturn.tech - View historical data on your bandwidth contributions, FIL earnings, and more.
- https://orchestrator.strn.pl/stats - View detailed, real-time stats on every Saturn node.
- https://dashboard.saturn.tech - View detailed, real-time stats on every Saturn node.
- https://dashboard.saturn.tech/address - View historical data on your bandwidth contributions, FIL earnings, and more.
- https://explorer.saturn.tech - View a 3D geospatial visualization of the Saturn network, along with nodes and network statistics.

### Community Tools

These Saturn tools are maintained by community members outside the Saturn core team.

- https://github.com/31z4/saturn-moonlet - Self-hosted Saturn monitoring using Prometheus and Grafana. View detailed, real-time and historical data on your nodes and earnings, setup alerts, and more.

## License

Expand Down
33 changes: 18 additions & 15 deletions container/nginx/conf.d/shared.conf
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,26 @@ location = / {
return 302 https://saturn.tech;
}

location ~ ^/(ipns|api)/ {
proxy_pass https://ipfs.io;

if ($request_method = 'OPTIONS') {
add_header 'Timing-Allow-Origin' '*';
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
return 204;
}
location / {
js_var $jwt;
js_content auth.routeRequest;
}

location / {
js_content badbits.filterCID;
location @auth_node_backend {
js_content auth.isAllowedRequest;

auth_jwt $jwt;
auth_jwt_key /etc/nginx/jwt_pub.key file;
auth_jwt_alg ES256;

# These headers are sent if the request fails auth.
add_header 'Saturn-Node-Id' '$node_id' always;
add_header 'Saturn-Transfer-Id' $request_id always;
add_header 'Timing-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Traceparent' always;
add_header 'Access-Control-Expose-Headers' '*' always;
}

location @node_backend {
Expand Down
1 change: 1 addition & 0 deletions container/nginx/confs/tls_proxy.conf
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ server {

include /usr/src/app/shared/nginx_conf/*.conf;
include /etc/nginx/conf.d/shared.conf;
include /etc/nginx/conf.d/register.conf;
}

server {
Expand Down
4 changes: 4 additions & 0 deletions container/nginx/jwt_pub.key
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEiVrZNLTZgPFkyBXI2MDM13e+tmKf
w82SnU183R6CczlsjO4qCTp3Xni+jBUri/5Ng34GZQfljtzZfDMfo2hHRw==
-----END PUBLIC KEY-----
3 changes: 2 additions & 1 deletion container/nginx/nginx.conf
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
load_module /usr/lib/nginx/modules/ngx_http_brotli_filter_module.so;
load_module /usr/lib/nginx/modules/ngx_http_brotli_static_module.so;
load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
load_module /usr/lib/nginx/modules/ngx_http_auth_jwt_module.so;

user nginx;
worker_processes auto;
Expand All @@ -15,7 +16,7 @@ events {
http {
js_path "/etc/nginx/njs/";
js_preload_object denylist.json;
js_import badbits from badbits.js;
js_import auth from auth.js;
js_import ipfsResponse from ipfs-response.js;

include /etc/nginx/mime.types;
Expand Down
88 changes: 88 additions & 0 deletions container/nginx/njs/auth.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import crypto from "crypto";

const ipfsRegex = /^\/ipfs\/(\w+)(\/?.*)/;

function routeRequest(req) {
const jwt = findJWT(req);
if (jwt) {
req.variables.jwt = jwt;
return req.internalRedirect("@auth_node_backend");
} else {
return req.internalRedirect("@node_backend");
}
}

function isAllowedRequest(req) {
const matches = req.uri.match(ipfsRegex);
if (!matches) {
return req.internalRedirect("@node_backend");
}
const cid = matches[1];

if (isBadBitsCid(cid)) {
return req.return(410);
}

if (!isAllowedDomain(req)) {
return req.return(403);
}

req.internalRedirect("@node_backend");
}

// TODO implement matching CID paths
// TODO convert CID v0 to CID v1
// implementation ref: https://github.com/protocol/bifrost-infra/blob/af46340bd830728b38a0ea632ca517d04277f78c/ansible/roles/nginx_conf_denylist/files/lua/helpers.lua#L80
function isBadBitsCid(cid) {
// check if root hash(`CID/`) is blocked via denylist.json
const hashedCID = crypto
.createHash("sha256")
.update(cid + "/")
.digest("hex");

/* eslint-disable-next-line no-undef */
return hashedCID in denylist;
}

function isAllowedDomain(req) {
const allowListStr = req.variables.jwt_claim_allow_list;
if (!allowListStr) {
return false;
}

let allowList;
try {
allowList = JSON.parse(allowListStr);
} catch (err) {
return false;
}

if (allowList.includes("*")) {
return true;
}

// Only browser requests are allowed for now.
const requestOrigin = req.variables.http_origin;
if (!requestOrigin) {
return false;
}
const requestDomain = requestOrigin.replace(/^https?:\/\//, "");

const isAllowedDomain = allowList.some((domain) => domain === requestDomain);

return isAllowedDomain;
}

function findJWT(req) {
const jwtQuery = req.variables.arg_jwt;

let jwtHeader = "";
const authHeader = req.variables.http_authorization;
if (authHeader) {
jwtHeader = authHeader.replace("Bearer ", "");
}

return jwtQuery || jwtHeader;
}

export default { routeRequest, isAllowedRequest, findJWT };
Loading

0 comments on commit 569ae65

Please sign in to comment.