Traefik reverse proxy with pathprefix attribute

Percona Monitoring and Management (PMM) PMM 2.x
, ,
1

Hi guys,

Hope you are all well !

I need to use traefik and to create a path like /admin/pmm-server to access the service.

For doing so, I thought to override your nginx configuration, mainly by adding a variable for the prefix_path, but it is not working properly.

here is my nginx configuration, any thoughs about how to fix it ?

  gzip on;
  etag on;


  upstream managed-grpc {
    server 127.0.0.1:7771;
    keepalive 32;
  }
  upstream managed-json {
    server 127.0.0.1:7772;
    keepalive 32;
    keepalive_requests 100;
    keepalive_timeout 75s;
  }


  upstream qan-api-grpc {
    server 127.0.0.1:9911;
    keepalive 32;
  }
  upstream qan-api-json {
    server 127.0.0.1:9922;
    keepalive 32;
    keepalive_requests 100;
    keepalive_timeout 75s;
  }


  server {


    set $prefix_path "/admin/pmm-server";


    listen 80;
    listen 443 ssl http2;
    server_name _;
    server_tokens off;


    # workaround CVE-2017-7529
    max_ranges 1;
    # allow huge requests
    large_client_header_buffers 128 64k;


    client_max_body_size 10m;


    ssl_certificate /srv/nginx/certificate.crt;
    ssl_certificate_key /srv/nginx/certificate.key;
    ssl_trusted_certificate /srv/nginx/ca-certs.pem;
    ssl_dhparam /srv/nginx/dhparam.pem;


    # Enable auth_request for all locations, including root
    # (but excluding /auth_request and /setup below).
    auth_request $prefix_path/auth_request;


    # Store the value of X-Must-Setup header of auth_request subrequest response in the variable.
    # It is used below in /auth_request.
    auth_request_set $auth_request_must_setup $upstream_http_x_must_setup;


    # nginx completely ignores auth_request subrequest response body.
    # We use that directive to send the same request to the same location as a normal request
    # to get a response body or redirect and return it to the client.
    # auth_request supports only 401 and 403 statuses; 401 is reserved for this configration,
    # and 403 is used for normal pmm-managed API errors.
    error_page 401 = $prefix_path/auth_request;


    # Internal location for authentication via pmm-managed/Grafana.
    # First, nginx sends request there to authenticate it. If it is not authenticated by pmm-managed/Grafana,
    # it is sent to this location for the second time (as a normal request) by error_page directive above.
    location $prefix_path/auth_request {
      internal;


      auth_request off;


      proxy_pass http://managed-json/auth_request;


      # nginx always strips body from authentication subrequests.
      # Overwrite Content-Length to avoid problems on Go side and to keep connection alive.
      proxy_pass_request_body off;
      proxy_set_header Content-Length 0;


      proxy_http_version 1.1;
      proxy_set_header Connection "";


      # This header is set only for to the second request, not for the first subrequest.
      # That variable is set above.
      proxy_set_header X-Must-Setup $auth_request_must_setup;


      # Those headers are set for both subrequest and normal request.
      proxy_set_header X-Original-Uri $request_uri;
      proxy_set_header X-Original-Method $request_method;
    }


    # AWS setup wizard
    location $prefix_path/setup {
      auth_request off;
      alias /usr/share/pmm-server/installation-wizard-page;
      try_files $uri $prefix_path/index.html break;
    }


    # Grafana
    rewrite ^$prefix_path/$ $scheme://$http_host$prefix_path/graph/;
    rewrite ^$prefix_path/graph$ $prefix_path/graph/;
    location $prefix_path/graph {
      proxy_cookie_path / "$prefix_path/;";
      proxy_pass http://127.0.0.1:3000;
      rewrite ^$prefix_path/graph/(.*) $prefix_path/$1 break;
      proxy_read_timeout 600;
    }


    # Remove next lines if bug is fixed https://github.com/grafana/grafana/issues/27226
    rewrite ^$prefix_path/(user/password/send-reset-email)$ $scheme://$http_host$prefix_path/graph/$1;
    rewrite ^$prefix_path/(login)$ $scheme://$http_host$prefix_path/graph/$1;


    # Remove next line if bug is fixed https://github.com/grafana/grafana/issues/27588
    rewrite ^$prefix_path/explore(.*)$ $scheme://$http_host/graph/explore$1;


    # Prometheus
    location $prefix_path/prometheus {
      proxy_pass http://127.0.0.1:9090;
      proxy_read_timeout 600;
    }


    # VictoriaMetrics
    location $prefix_path/victoriametrics/ {
      proxy_pass http://127.0.0.1:9090/prometheus/;
      proxy_read_timeout 600;
    }


    # VMAlert
    location $prefix_path/prometheus/rules {
      proxy_pass http://127.0.0.1:8880/api/v1/groups;
      proxy_read_timeout 600;
    }
    location $prefix_path/prometheus/alerts {
      proxy_pass http://127.0.0.1:8880/api/v1/alerts;
      proxy_read_timeout 600;
    }


    # Alertmanager
    location $prefix_path/alertmanager {
      proxy_pass http://127.0.0.1:9093;
    }


    # QAN App
    location $prefix_path/qan {
      alias /usr/share/percona-qan-app;
      try_files $uri $prefix_path/index.html break;
      add_header X-Frame-Options SAMEORIGIN;
    }


    # Swagger UI
    rewrite ^$prefix_path/swagger$ $prefix_path$1/swagger/;
    rewrite ^$prefix_path/swagger.json$ $prefix_path/swagger/swagger.json;
    location $prefix_path/swagger {
      alias /usr/share/pmm-server/swagger/;
      try_files $uri /swagger-ui.html break;
    }


    # pmm-managed gRPC APIs
    location $prefix_path/agent. {
      grpc_pass grpc://managed-grpc;
      # Disable request body size check for gRPC streaming, see https://trac.nginx.org/nginx/ticket/1642.
      # pmm-managed uses grpc.MaxRecvMsgSize for that.
      client_max_body_size 0;
    }
    location $prefix_path/inventory. {
      grpc_pass grpc://managed-grpc;
    }
    location $prefix_path/management. {
      grpc_pass grpc://managed-grpc;
    }
    location $prefix_path/server. {
      grpc_pass grpc://managed-grpc;
    }


    # pmm-managed JSON APIs
    location $prefix_path/v1/ {
      proxy_pass http://managed-json/v1/;
      proxy_http_version 1.1;
      proxy_set_header Connection "";
    }


    # qan-api gRPC APIs should not be exposed


    # qan-api JSON APIs
    location $prefix_path/v0/qan/ {
      proxy_pass http://qan-api-json/v0/qan/;
      proxy_http_version 1.1;
      proxy_set_header Connection "";
    }


    # for minimal compatibility with PMM 1.x
    rewrite ^$prefix_path/ping$ $prefix_path/v1/readyz;
    rewrite ^$prefix_path/managed/v1/version$ $prefix_path/v1/version;


    # logs.zip in both PMM 1.x and 2.x variants
    rewrite ^$prefix_path/managed/logs.zip$ $prefix_path/logs.zip;
    location $prefix_path/logs.zip {
      proxy_pass http://managed-json;
      proxy_http_version 1.1;
      proxy_set_header Connection "";
    }


    # This localtion stores static content for general pmm-server purposes.
    # Ex.: local-rss.xml - contains Percona's news when no internet connection.
    location $prefix_path/pmm-static {
      auth_request off;
      alias /usr/share/pmm-server/static;
    }


    # proxy requests to the Percona's blog feed
    # fallback to local rss if pmm-server is isolated from internet.
    # https://jira.percona.com/browse/PMM-6153
    location = $prefix_path/percona-blog/feed {
      auth_request off;


      set $feed https://www.percona.com/blog/feed/;
      proxy_pass $feed;
      proxy_set_header User-Agent "$http_user_agent pmm-server/2.x";
      error_page 500 502 503 504 $prefix_path/pmm-static/local-rss.xml;
    }


  }
2

Here is the excerpt of my docker-compose/traefik configuration:

  pmm-server:

    image: percona/pmm-server:${PERCONA_PMM_SERVER_VERSION}

    container_name: pmm-server

    hostname: pmm-server

    logging:

      driver: json-file

      options:

        max-size: "10m"

        max-file: "5"

    expose:

    - "443"

    - "80"

    - "8080"

    ports:

    - "80:80"

    # - "8443:443"

    networks:

    - web

    - internal

    volumes_from:

    - pmm-data

    volumes:

    - ./shared/config/pmm-server/nginx.conf:/etc/nginx/conf.d/pmm.conf

    - ./shared/logs/pmm-server:/srv/logs:ro

    labels:    

    - "traefik.enable=true"

    - "traefik.docker.network=web"

    - "traefik.http.routers.pmm-server.rule=Host(`xxxx-xxxx.xxx-xxxxxx.com`) && PathPrefix(`/admin/pmm-server`)"

    - 'traefik.http.routers.pmm-server.tls=true'

    - "traefik.http.routers.pmm-server.service=pmm-server@docker"

    - 'traefik.http.routers.pmm-server.tls.certresolver=letsencrypt'

    - "traefik.http.routers.pmm-server.entrypoints=https"

    - "traefik.http.services.pmm-server.loadbalancer.server.port=80"

    restart: unless-stopped
3

Hi guys,

Hope you are all well !

Can you advise me on this one because it is important for any future deployment of pmm-server on my cluster of e-shops ?

Thanks a lot in advance.

Cheers,

Luc Michalski

4

Hi Luc,

nginx that is shipped with PMM is a reverse proxy itself. While in theory it seems like possible to put another proxy (traefik in your case) in front of nginx, I doubt that there is currently an easy way to make it work.

The major problem comes from that fact we don’t offer an nginx.conf variable (like the one you are trying to implement) that you could use to override the base/root path, therefore there are too many places where you need to patch the config, which is very cumbersome and error-prone. The other problem is that all our dashboards have hard-codes paths that start with /graph/... which means they would only render when the base/root path also starts with /graph. Example: https://github.com/percona/grafana-dashboards/blob/PMM-2.0/dashboards/pmm-settings.json#L30

Sorry that this doesn’t solve your problem.

Please feel free to create a ticket to raise this issue.