Add LDAP user data caching using redis

.example.env

@@ -42,3 +42,4 @@ BACKEND_LDAP_BASE_DN="dc=ldap,dc=example,dc=com"
 BACKEND_LDAP_BASE_USER_DN="ou=users,dc=ldap,dc=example,dc=com"
 BACKEND_LDAP_BIND_DN="cn=admin,dc=ldap,dc=example,dc=com"
 BACKEND_LDAP_BIND_PASSWORD=
+BACKEND_LDAP_CACHE_REFRESH_INTERVAL=60

docker-compose.yml

@@ -79,7 +79,6 @@ services:
       BACKEND_URL: ${URL}
       BACKEND_API_GRAPHQL_PLAYGROUND: ${BACKEND_API_GRAPHQL_PLAYGROUND}
       BACKEND_API_JWT_SECRET: ${BACKEND_API_JWT_SECRET}
-      BACKEND_WORKER_REDIS_URL: redis://redis:6379
       BACKEND_SMTP_HELO: ${BACKEND_SMTP_HELO}
       BACKEND_SMTP_HOST: ${BACKEND_SMTP_HOST}
       BACKEND_SMTP_PORT: ${BACKEND_SMTP_PORT}
@@ -87,12 +86,15 @@ services:
       BACKEND_SMTP_USERNAME: ${BACKEND_SMTP_USERNAME}
       BACKEND_SMTP_PASSWORD: ${BACKEND_SMTP_PASSWORD}
       BACKEND_DB_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_USER}
+      BACKEND_REDIS_HOST: redis
+      BACKEND_REDIS_PORT: 6379
       BACKEND_LDAP_HOST: ${BACKEND_LDAP_HOST}
       BACKEND_LDAP_PORT: ${BACKEND_LDAP_PORT}
       BACKEND_LDAP_BASE_DN: ${BACKEND_LDAP_BASE_DN}
       BACKEND_LDAP_BASE_USER_DN: ${BACKEND_LDAP_BASE_USER_DN}
       BACKEND_LDAP_BIND_DN: ${BACKEND_LDAP_BIND_DN}
       BACKEND_LDAP_BIND_PASSWORD: ${BACKEND_LDAP_BIND_PASSWORD}
+      BACKEND_LDAP_CACHE_REFRESH_INTERVAL: ${BACKEND_LDAP_CACHE_REFRESH_INTERVAL}
 
   frontend:
     build:

docker/backend/shard.yml

@@ -68,3 +68,5 @@ dependencies:
     github: Sija/retriable.cr
   service:
     git: https://git.dergrimm.net/dergrimm/service.git
+  redis:
+    github: stefanwille/crystal-redis

docker/backend/src/backend/api/schema/mutation.cr

@@ -27,7 +27,7 @@ module Backend
           raise "Auth failed" if username.empty? || password.empty?
 
           user = Db::User.find_by(username: username)
-          raise "Auth failed" unless user && Ldap.authenticate?(Ldap.uid(username), password)
+          raise "Auth failed" unless user && Ldap.authenticate?(Ldap::Constructor.uid(username), password)
 
           LoginPayload.new(
             user: User.new(user),
@@ -44,7 +44,7 @@ module Backend
           context.admin!
 
           raise "LDAP user does not exist" if check_ldap && begin
-                                                !Ldap.user(Ldap.uid(input.username))
+                                                !Ldap.user(Ldap::Constructor.uid(input.username))
                                               rescue LDAP::Client::AuthError
                                                 true
                                               end

docker/backend/src/backend/config.cr

@@ -63,10 +63,6 @@ module Backend
     # Configuration for `Api`
     getter api : ApiConfig
 
-    @[EnvConfig::Setting(key: "worker")]
-    # Configuration for `Worker`
-    getter worker : WorkerConfig
-
     @[EnvConfig::Setting(key: "smtp")]
     # Configuration for `Mailers`
     getter smtp : SmtpConfig
@@ -75,6 +71,9 @@ module Backend
     # Configuration for `Db`
     getter db : DbConfig
 
+    @[EnvConfig::Setting(key: "redis")]
+    getter redis : RedisConfig
+
     @[EnvConfig::Setting(key: "ldap")]
     # Configuration for `Ldap`
     getter ldap : LdapConfig
@@ -97,14 +96,6 @@ module Backend
       end
     end
 
-    # Configuration for `Worker`
-    class WorkerConfig
-      include EnvConfig
-
-      # Redis URL
-      getter redis_url : String
-    end
-
     # Configuration for `Mailers`
     class SmtpConfig
       include EnvConfig
@@ -138,6 +129,22 @@ module Backend
       getter url : String
     end
 
+    # Configuration for `REDIS`
+    class RedisConfig
+      include EnvConfig
+
+      # Redis host
+      getter host : String
+
+      # Redis port
+      getter port : Int32
+
+      # Redis URL
+      def url : String
+        "redis://#{host}:#{port}"
+      end
+    end
+
     # Configuration for `Ldap`
     class LdapConfig
       include EnvConfig
@@ -161,6 +168,9 @@ module Backend
 
       # LDAP bind password
       getter bind_password : String
+
+      # Periodical cache refresh interval
+      getter cache_refresh_interval : Int32
     end
   end
 end

docker/backend/src/backend/db/user.cr

@@ -20,6 +20,9 @@ module Backend
     class User < Granite::Base
       table users
 
+      # LDAP user data
+      getter ldap : Ldap::User?
+
       has_one :teacher
       has_one :student
 
@@ -37,22 +40,27 @@ module Backend
 
       # User's first name
       def first_name : String
-        Ldap.user(Ldap.uid(@username.not_nil!)).first["givenName"].first
+        refresh_ldap.first_name
       end
 
       # User's last name
       def last_name : String
-        Ldap.user(Ldap.uid(@username.not_nil!)).first["sn"].first
+        refresh_ldap.last_name
       end
 
       # User's full name
       def name : String
-        Ldap.user(Ldap.uid(@username.not_nil!)).first["cn"].first
+        "#{first_name} #{last_name}"
       end
 
       # User's email
       def email : String
-        Ldap.user(Ldap.uid(@username.not_nil!)).first["mail"].first
+        refresh_ldap.email
+      end
+
+      # Refreshes LDAP user data
+      def refresh_ldap : Ldap::User
+        (@ldap ||= Ldap::User.from_json(REDIS.get("ldap:user:#{@id}").as(String))).not_nil!
       end
 
       validate :role, "needs to be a valid role" do |user|

docker/backend/src/backend/ldap.cr

@@ -18,6 +18,8 @@ require "ldap"
 require "socket"
 require "ldap_escape"
 
+require "./ldap/*"
+
 module Backend
   # Provides LDAP utility functions
   module Ldap
@@ -28,23 +30,19 @@ module Backend
       LDAP::Client.new(TCPSocket.new(Backend.config.ldap.host, Backend.config.ldap.port))
     end
 
-    # Constructs a CN DN from a username
-    def cn(username : String) : String
-      "cn=#{LdapEscape.dn(username)},#{Backend.config.ldap.user_dn}"
-    end
-
-    # Constructs a UID DN from a username
-    def uid(uid : String) : String
-      "uid=#{LdapEscape.dn(uid)},#{Backend.config.ldap.base_user_dn}"
-    end
-
     # Queries the LDAP server for a user
     #
-    # NOTE: Returns a hash of the user's attributes
-    def user(dn : String) : Array(Hash(String, Array(String)))
+    # NOTE: Returns raw LDAP data
+    def raw_user(dn : String) : User::Raw
       create_client
         .authenticate(Backend.config.ldap.bind_dn, Backend.config.ldap.bind_password)
         .search(base: dn)
+        .first
+    end
+
+    # Queries the LDAP server for a user
+    def user(dn : String) : User
+      User.from_raw(raw_user(dn))
     end
 
     # Checks if credentials are valid

docker/backend/src/backend/ldap/constructor.cr

@@ -0,0 +1,18 @@
+module Backend
+  module Ldap
+    # DN construction utilities
+    module Constructor
+      extend self
+
+      # Constructs a CN DN from a username
+      def cn(username : String) : String
+        "cn=#{LdapEscape.dn(username)},#{Backend.config.ldap.user_dn}"
+      end
+
+      # Constructs a UID DN from a username
+      def uid(uid : String) : String
+        "uid=#{LdapEscape.dn(uid)},#{Backend.config.ldap.base_user_dn}"
+      end
+    end
+  end
+end

docker/backend/src/backend/ldap/user.cr

@@ -0,0 +1,36 @@
+require "json"
+
+module Backend
+  module Ldap
+    # LDAP user properties
+    struct User
+      include JSON::Serializable
+
+      alias Raw = Hash(String, Array(String))
+
+      @[JSON::Field(key: "givenName")]
+      # First name
+      property first_name : String
+
+      @[JSON::Field(key: "sn")]
+      # Last name
+      property last_name : String
+
+      @[JSON::Field(key: "mail")]
+      # Email address
+      property email : String
+
+      def initialize(@first_name : String, @last_name : String, @email : String)
+      end
+
+      # Creates user data from LDAP entry
+      def self.from_raw(raw : Raw) : self
+        self.new(
+          first_name: raw["givenName"].first,
+          last_name: raw["sn"].first,
+          email: raw["mail"].first
+        )
+      end
+    end
+  end
+end

docker/backend/src/backend/redis.cr

@@ -0,0 +1,5 @@
+require "redis"
+
+module Backend
+  REDIS = Redis::PooledClient.new(host: config.redis.host, port: config.redis.port)
+end

docker/backend/src/backend/worker.cr

@@ -37,7 +37,7 @@ module Backend
   # Worker module
   module Worker
     Mosquito.configure do |settings|
-      settings.redis_url = Backend.config.worker.redis_url
+      settings.redis_url = Backend.config.redis.url
     end
   end
 end

docker/backend/src/backend/worker/jobs/cache_ldap_users_job.cr

@@ -0,0 +1,47 @@
+# Mentorenwahl: A fullstack application for assigning mentors to students based on their whishes.
+# Copyright (C) 2022  Dominic Grimm
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+module Backend
+  module Worker
+    module Jobs
+      # Peridically caches user data in redis cache
+      class CacheLdapUsersJob < Mosquito::PeriodicJob
+        run_every Backend.config.ldap.cache_refresh_interval.minutes
+
+        # :ditto:
+        def perform : Nil
+          REDIS.keys("ldap:user:*")
+            .map(&.as(String).split(":")[2].to_i64)
+            .concat(Db::User.all.map(&.id.not_nil!))
+            .uniq!
+            .each do |id|
+              spawn do
+                key = "ldap:user:#{id}"
+                user = Db::User.find(id)
+                if user
+                  REDIS.set(key, user.refresh_ldap.to_json, (Backend.config.ldap.cache_refresh_interval * 2).minutes.to_i)
+                else
+                  REDIS.del(key)
+                end
+              end
+            end
+
+          Fiber.yield
+        end
+      end
+    end
+  end
+end

docker/backend/src/backend/worker/jobs/send_teachers_registration_email_job.cr

@@ -14,8 +14,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-require "../../db/user"
-
 module Backend
   module Worker
     module Jobs

docs/src/configuration.md

@@ -4,26 +4,27 @@
 
 To change the environment variables edit the `.env` file.
 
-| Name                             | Type     | Description                                                                          |
-| -------------------------------- | -------- | ------------------------------------------------------------------------------------ |
-| `URL`                            | `String` | The webroot of the application to base of links and routing                          |
-| `POSTGRES_USER`                  | `String` | Database user name                                                                   |
-| `POSTGRES_PASSWORD`              | `String` | Database password                                                                    |
-| `BACKEND_URL`                    | `String` | Backend webroot (= `URL`)                                                            |
-| `BACKEND_API_GRAPHQL_PLAYGROUND` | `Bool`   | Enables GraphQL playground (automatically enabled when compiled in development mode) |
-| `BACKEND_API_JWT_SECRET`         | `String` | Password to encrypt all authentication tokens with                                   |
-| `BACKEND_SMTP_HELO`              | `String` | SMTP server HELO                                                                     |
-| `BACKEND_SMTP_HOST`              | `String` | SMTP server hostname                                                                 |
-| `BACKEND_SMTP_PORT`              | `Int32`  | SMTP server port (= `587`)                                                           |
-| `BACKEND_SMTP_NAME`              | `String` | Name to send emails with                                                             |
-| `BACKEND_SMTP_USERNAME`          | `String` | SMTP account username                                                                |
-| `BACKEND_SMTP_PASSWORD`          | `String` | SMTP account password                                                                |
-| `BACKEND_LDAP_HOST`              | `String` | LDAP server hostname                                                                 |
-| `BACKEND_LDAP_PORT`              | `Int32`  | LDAP server port (= `389`)                                                           |
-| `BACKEND_LDAP_BASE_DN`           | `String` | LDAP virtual DN                                                                      |
-| `BACKEND_LDAP_BASE_USER_DN`      | `String` | LDAP user group DN                                                                   |
-| `BACKEND_LDAP_BIND_DN`           | `String` | LDAP admin account DN                                                                |
-| `BACKEND_LDAP_BIND_PASSWORD`     | `String` | LDAP admin account password                                                          |
+| Name                                  | Type     | Description                                                                          |
+| ------------------------------------- | -------- | ------------------------------------------------------------------------------------ |
+| `URL`                                 | `String` | The webroot of the application to base of links and routing                          |
+| `POSTGRES_USER`                       | `String` | Database user name                                                                   |
+| `POSTGRES_PASSWORD`                   | `String` | Database password                                                                    |
+| `BACKEND_URL`                         | `String` | Backend webroot (= `URL`)                                                            |
+| `BACKEND_API_GRAPHQL_PLAYGROUND`      | `Bool`   | Enables GraphQL playground (automatically enabled when compiled in development mode) |
+| `BACKEND_API_JWT_SECRET`              | `String` | Password to encrypt all authentication tokens with                                   |
+| `BACKEND_SMTP_HELO`                   | `String` | SMTP server HELO                                                                     |
+| `BACKEND_SMTP_HOST`                   | `String` | SMTP server hostname                                                                 |
+| `BACKEND_SMTP_PORT`                   | `Int32`  | SMTP server port (= `587`)                                                           |
+| `BACKEND_SMTP_NAME`                   | `String` | Name to send emails with                                                             |
+| `BACKEND_SMTP_USERNAME`               | `String` | SMTP account username                                                                |
+| `BACKEND_SMTP_PASSWORD`               | `String` | SMTP account password                                                                |
+| `BACKEND_LDAP_HOST`                   | `String` | LDAP server hostname                                                                 |
+| `BACKEND_LDAP_PORT`                   | `Int32`  | LDAP server port (= `389`)                                                           |
+| `BACKEND_LDAP_BASE_DN`                | `String` | LDAP virtual DN                                                                      |
+| `BACKEND_LDAP_BASE_USER_DN`           | `String` | LDAP user group DN                                                                   |
+| `BACKEND_LDAP_BIND_DN`                | `String` | LDAP admin account DN                                                                |
+| `BACKEND_LDAP_BIND_PASSWORD`          | `String` | LDAP admin account password                                                          |
+| `BACKEND_LDAP_CACHE_REFRESH_INTERVAL` | `Int32`  | Periodical cache refresh interval in minutes                                         |
 
 ## Compile time