Keycloak, Get Consents

.gitignore

@@ -8,6 +8,7 @@
 .settings
 .metals
 .vscode
+.zed
 .classpath
 .project
 .cache
@@ -30,8 +31,9 @@ obp-api/src/main/scala/code/api/v3_0_0/custom/
 marketing_diagram_generation/outputs/*
 
 .bloop
+!.bloop/*.json
 .bsp
 .specstory
 project/project
 coursier
-*.code-workspace
+*.code-workspace

.metals-config.json

@@ -0,0 +1,76 @@
+{
+  "maven": {
+    "enabled": true
+  },
+  "metals": {
+    "serverVersion": "1.0.0",
+    "javaHome": "/usr/lib/jvm/java-17-openjdk-amd64",
+    "bloopVersion": "2.0.0",
+    "superMethodLensesEnabled": true,
+    "enableSemanticHighlighting": true,
+    "compileOnSave": true,
+    "testUserInterface": "Code Lenses",
+    "inlayHints": {
+      "enabled": true,
+      "hintsInPatternMatch": {
+        "enabled": true
+      },
+      "implicitArguments": {
+        "enabled": true
+      },
+      "implicitConversions": {
+        "enabled": true
+      },
+      "inferredTypes": {
+        "enabled": true
+      },
+      "typeParameters": {
+        "enabled": true
+      }
+    }
+  },
+  "buildTargets": [
+    {
+      "id": "obp-commons",
+      "displayName": "obp-commons",
+      "baseDirectory": "file:///home/marko/Tesobe/GitHub/constantine2nd/OBP-API/obp-commons/",
+      "tags": ["library"],
+      "languageIds": ["scala", "java"],
+      "dependencies": [],
+      "capabilities": {
+        "canCompile": true,
+        "canTest": true,
+        "canRun": false,
+        "canDebug": true
+      },
+      "dataKind": "scala",
+      "data": {
+        "scalaOrganization": "org.scala-lang",
+        "scalaVersion": "2.12.20",
+        "scalaBinaryVersion": "2.12",
+        "platform": "jvm"
+      }
+    },
+    {
+      "id": "obp-api",
+      "displayName": "obp-api",
+      "baseDirectory": "file:///home/marko/Tesobe/GitHub/constantine2nd/OBP-API/obp-api/",
+      "tags": ["application"],
+      "languageIds": ["scala", "java"],
+      "dependencies": ["obp-commons"],
+      "capabilities": {
+        "canCompile": true,
+        "canTest": true,
+        "canRun": true,
+        "canDebug": true
+      },
+      "dataKind": "scala",
+      "data": {
+        "scalaOrganization": "org.scala-lang",
+        "scalaVersion": "2.12.20",
+        "scalaBinaryVersion": "2.12",
+        "platform": "jvm"
+      }
+    }
+  ]
+}

README.md

@@ -46,11 +46,22 @@ This project is dual licensed under the AGPL V3 (see NOTICE) and commercial lice
 The project uses Maven 3 as its build tool.
 
 To compile and run Jetty, install Maven 3, create your configuration in `obp-api/src/main/resources/props/default.props` and execute:
+To compile and run Jetty, install Maven 3, create your configuration in `obp-api/src/main/resources/props/`, copy `sample.props.template` to `default.props` and edit the latter. Then:
 
 ```sh
 mvn install -pl .,obp-commons && mvn jetty:run -pl obp-api
 ```
 
+### ZED IDE Setup
+
+For ZED IDE users, we provide a complete development environment with Scala language server support:
+
+```bash
+./zed/setup-zed-ide.sh
+```
+
+This sets up automated build tasks, code navigation, and real-time error checking. See [`zed/README.md`](zed/README.md) for complete documentation.
+
 In case the above command fails try the next one:
 
 ```sh
@@ -207,6 +218,7 @@ Once Postgres is installed (On macOS, use `brew`):
 1.  Grant all on database `obpdb` to `obp`; (So OBP-API can create tables etc.)
 
 #### For newer versions of postgres 16 and above, you need to follow the following instructions
+
 -- Connect to the sandbox database
 \c sandbox;
 

obp-api/src/main/resources/props/sample.props.template

@@ -1518,3 +1518,48 @@ regulated_entities = []
 
 
 # Note: For secure and http only settings for cookies see resources/web.xml which is mentioned in the README.md
+
+
+
+##########################################################
+#                      Redis Logging                     #
+##########################################################
+## Enable Redis logging (true/false)
+redis_logging_enabled = false
+
+## Batch size for sending logs to Redis
+## Smaller batch size reduces latency for logging critical messages.
+redis_logging_batch_size = 50
+
+## Flush interval for batch logs in milliseconds
+## Flush every 500ms to keep Redis queues up-to-date without too much delay.
+redis_logging_flush_interval_ms = 500
+
+## Maximum number of retries for failed log writes
+## Ensures transient Redis errors are retried before failing.
+redis_logging_max_retries = 3
+
+## Number of consecutive failures before opening circuit breaker
+## Prevents hammering Redis when it is down.
+redis_logging_circuit_breaker_threshold = 10
+
+## Number of threads for asynchronous Redis operations
+## Keep small for lightweight logging; adjust if heavy logging is expected.
+redis_logging_thread_pool_size = 2
+
+## SIX different FIFO Redis queues. Each queue has a maximum number of entries.
+## These control how many messages are kept per log level.
+## 1000 is a reasonable default; adjust if you expect higher traffic.
+redis_logging_trace_queue_max_entries = 1000     # Max TRACE messages
+redis_logging_debug_queue_max_entries = 1000     # Max DEBUG messages
+redis_logging_info_queue_max_entries = 1000      # Max INFO messages
+redis_logging_warning_queue_max_entries = 1000   # Max WARNING messages
+redis_logging_error_queue_max_entries = 1000     # Max ERROR messages
+redis_logging_all_queue_max_entries = 1000       # Max ALL messages
+
+## Optional: Circuit breaker reset interval (ms)
+## How long before retrying after circuit breaker opens. Default 60s.
+redis_logging_circuit_breaker_reset_ms = 60000
+##########################################################
+#                      Redis Logging                     #
+##########################################################

obp-api/src/main/scala/bootstrap/liftweb/CustomDBVendor.scala

@@ -1,6 +1,7 @@
 package bootstrap.liftweb
 
 import code.api.util.APIUtil
+import code.util.Helper.MdcLoggable
 import com.zaxxer.hikari.pool.ProxyConnection
 import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
 
@@ -21,19 +22,17 @@ import net.liftweb.util.Helpers.tryo
 class CustomDBVendor(driverName: String,
                      dbUrl: String,
                      dbUser: Box[String],
-                     dbPassword: Box[String]) extends CustomProtoDBVendor {
-
-  private val logger = Logger(classOf[CustomDBVendor])
+                     dbPassword: Box[String]) extends CustomProtoDBVendor with MdcLoggable {
 
   object HikariDatasource {
     val config = new HikariConfig()
-    
+
     val connectionTimeout = APIUtil.getPropsAsLongValue("hikari.connectionTimeout")
     val maximumPoolSize = APIUtil.getPropsAsIntValue("hikari.maximumPoolSize")
     val idleTimeout = APIUtil.getPropsAsLongValue("hikari.idleTimeout")
     val keepaliveTime = APIUtil.getPropsAsLongValue("hikari.keepaliveTime")
     val maxLifetime = APIUtil.getPropsAsLongValue("hikari.maxLifetime")
-    
+
     if(connectionTimeout.isDefined){
       config.setConnectionTimeout(connectionTimeout.head)
     }
@@ -63,7 +62,7 @@ class CustomDBVendor(driverName: String,
       case _ =>
         config.setJdbcUrl(dbUrl)
     }
-    
+
     config.addDataSourceProperty("cachePrepStmts", "true")
     config.addDataSourceProperty("prepStmtCacheSize", "250")
     config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048")
@@ -79,8 +78,7 @@ class CustomDBVendor(driverName: String,
   def closeAllConnections_!(): Unit = HikariDatasource.ds.close()
 }
 
-trait CustomProtoDBVendor extends ConnectionManager {
-  private val logger = Logger(classOf[CustomProtoDBVendor])
+trait CustomProtoDBVendor extends ConnectionManager with MdcLoggable {
 
   def createOne: Box[Connection]
 
@@ -90,4 +88,4 @@ trait CustomProtoDBVendor extends ConnectionManager {
 
   def releaseConnection(conn: Connection): Unit = {conn.asInstanceOf[ProxyConnection].close()}
 
-}
+}

obp-api/src/main/scala/code/accountattribute/AccountAttribute.scala

@@ -7,6 +7,7 @@ import com.openbankproject.commons.model.enums.AccountAttributeType
 import com.openbankproject.commons.model.{AccountAttribute, AccountId, BankId, BankIdAccountId, ProductAttribute, ProductCode, ViewId}
 import net.liftweb.common.{Box, Logger}
 import net.liftweb.util.SimpleInjector
+import code.util.Helper.MdcLoggable
 
 import scala.collection.immutable.List
 import scala.concurrent.Future
@@ -16,7 +17,7 @@ object AccountAttributeX extends SimpleInjector {
   val accountAttributeProvider = new Inject(buildOne _) {}
 
   def buildOne: AccountAttributeProvider = MappedAccountAttributeProvider
-  
+
   // Helper to get the count out of an option
   def countOfAccountAttribute(listOpt: Option[List[AccountAttribute]]): Int = {
     val count = listOpt match {
@@ -29,17 +30,15 @@ object AccountAttributeX extends SimpleInjector {
 
 }
 
-trait AccountAttributeProvider {
-
-  private val logger = Logger(classOf[AccountAttributeProvider])
+trait AccountAttributeProvider extends MdcLoggable {
 
   def getAccountAttributesFromProvider(accountId: AccountId, productCode: ProductCode): Future[Box[List[AccountAttribute]]]
   def getAccountAttributesByAccount(bankId: BankId,
                                     accountId: AccountId): Future[Box[List[AccountAttribute]]]
-  def getAccountAttributesByAccountCanBeSeenOnView(bankId: BankId, 
-                                                   accountId: AccountId, 
+  def getAccountAttributesByAccountCanBeSeenOnView(bankId: BankId,
+                                                   accountId: AccountId,
                                                    viewId: ViewId): Future[Box[List[AccountAttribute]]]
-  def getAccountAttributesByAccountsCanBeSeenOnView(accounts: List[BankIdAccountId], 
+  def getAccountAttributesByAccountsCanBeSeenOnView(accounts: List[BankIdAccountId],
                                                     viewId: ViewId): Future[Box[List[AccountAttribute]]]
 
   def getAccountAttributeById(productAttributeId: String): Future[Box[AccountAttribute]]
@@ -58,10 +57,10 @@ trait AccountAttributeProvider {
                               productCode: ProductCode,
                               accountAttributes: List[ProductAttribute],
                               productInstanceCode: Option[String]): Future[Box[List[AccountAttribute]]]
-  
+
   def deleteAccountAttribute(accountAttributeId: String): Future[Box[Boolean]]
 
   def getAccountIdsByParams(bankId: BankId, params: Map[String, List[String]]): Future[Box[List[String]]]
 
   // End of Trait
-}
+}

obp-api/src/main/scala/code/api/OAuth2.scala

@@ -328,18 +328,26 @@ object OAuth2Login extends RestHelper with MdcLoggable {
     def getOrCreateResourceUser(idToken: String): Box[User] = {
       val uniqueIdGivenByProvider = JwtUtil.getSubject(idToken).getOrElse("")
       val provider = resolveProvider(idToken)
-      Users.users.vend.getUserByProviderId(provider = provider, idGivenByProvider = uniqueIdGivenByProvider).or { // Find a user
-        Users.users.vend.createResourceUser( // Otherwise create a new one
-          provider = provider,
-          providerId = Some(uniqueIdGivenByProvider),
-          None,
-          name = getClaim(name = "given_name", idToken = idToken).orElse(Some(uniqueIdGivenByProvider)),
-          email = getClaim(name = "email", idToken = idToken),
-          userId = None,
-          createdByUserInvitationId = None,
-          company = None,
-          lastMarketingAgreementSignedDate = None
-        )
+      KeycloakFederatedUserReference.parse(uniqueIdGivenByProvider) match {
+        case Right(fedRef) => // Users log on via Keycloak, which uses User Federation to access the external OBP database.
+          logger.debug(s"External ID = ${fedRef.externalId}")
+          logger.debug(s"Storage Provider ID = ${fedRef.storageProviderId}")
+          Users.users.vend.getUserByResourceUserId(fedRef.externalId)
+        case Left(error) =>
+          logger.debug(s"Parse error: $error")
+          Users.users.vend.getUserByProviderId(provider = provider, idGivenByProvider = uniqueIdGivenByProvider).or { // Find a user
+            Users.users.vend.createResourceUser( // Otherwise create a new one
+              provider = provider,
+              providerId = Some(uniqueIdGivenByProvider),
+              None,
+              name = getClaim(name = "given_name", idToken = idToken).orElse(Some(uniqueIdGivenByProvider)),
+              email = getClaim(name = "email", idToken = idToken),
+              userId = None,
+              createdByUserInvitationId = None,
+              company = None,
+              lastMarketingAgreementSignedDate = None
+            )
+          }
       }
     }
 
@@ -526,7 +534,7 @@ object OAuth2Login extends RestHelper with MdcLoggable {
       */
     override def wellKnownOpenidConfiguration: URI =
       new URI(
-        APIUtil.getPropsValue(nameOfProperty = "oauth2.keycloak.well_known", "http://localhost:7070/realms/master/.well-known/openid-configuration")
+        APIUtil.getPropsValue(nameOfProperty = "oauth2.keycloak.well_known", "http://localhost:8000/realms/master/.well-known/openid-configuration")
       )
     override def urlOfJwkSets: Box[String] = checkUrlOfJwkSets(identityProvider = keycloakHost)
     def isIssuer(jwt: String): Boolean = isIssuer(jwtToken=jwt, identityProvider = keycloakHost)

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala

@@ -29,6 +29,7 @@ import com.openbankproject.commons.model._
 import com.openbankproject.commons.model.enums.TransactionRequestTypes._
 import com.openbankproject.commons.model.enums.{AttributeCategory, CardAttributeType, ChallengeType, TransactionRequestStatus}
 import com.openbankproject.commons.util.{ApiVersion, FieldNameApiVersions, ReflectUtils}
+import net.liftweb.common.Full
 import net.liftweb.json
 
 import java.net.URLEncoder
@@ -4255,6 +4256,8 @@ object SwaggerDefinitionsJSON {
     consent_reference_id = consentReferenceIdExample.value,
     consumer_id = consumerIdExample.value,
     created_by_user_id = userIdExample.value,
+    provider = Some(providerValueExample.value),
+    provider_id = Some(providerIdExample.value),
     last_action_date = dateExample.value,
     last_usage_date = dateTimeExample.value,
     status = ConsentStatus.INITIATED.toString,
@@ -4282,7 +4285,7 @@ object SwaggerDefinitionsJSON {
     consents =  List(consentInfoJsonV510)
   )
 
-  lazy val consentsJsonV510 = ConsentsJsonV510(List(allConsentJsonV510))
+  lazy val consentsJsonV510 = ConsentsJsonV510(number_of_rows = 1, List(allConsentJsonV510))
 
   lazy val revokedConsentJsonV310 = ConsentJsonV310(
     consent_id = "9d429899-24f5-42c8-8565-943ffa6a7945",

obp-api/src/main/scala/code/api/cache/Redis.scala

@@ -57,6 +57,22 @@ object Redis extends MdcLoggable {
 
   def jedisPoolDestroy: Unit = jedisPool.destroy()
 
+  def isRedisReady: Boolean = {
+    var jedisConnection: Option[Jedis] = None
+    try {
+      jedisConnection = Some(jedisPool.getResource)
+      val pong = jedisConnection.get.ping() // sends PING command
+      pong == "PONG"
+    } catch {
+      case e: Throwable =>
+        logger.error(s"Redis is not ready: ${e.getMessage}")
+        false
+    } finally {
+      jedisConnection.foreach(_.close())
+    }
+  }
+
+
   private def configureSslContext(): SSLContext = {
 
     // Load the CA certificate