Merge pull request #48 from shreelakshmijoshi/add-adapter-configs

Update RS Proxy

.gitignore

@@ -9,6 +9,7 @@ docker-compose.temp.yml
 config.properties
 upload-dir/
 temp-dir/
+examples/consumers(adapter)/secrets/
 .idea
 iudx.rs.proxy.iml
 ### STS ###

README.md

@@ -4,11 +4,11 @@
 
 # iudx-resource-proxy-server
 
-The resource server proxy is [IUDXs](https://iudx.org.in) data discovery portal.
+The resource server proxy is [Data Exchange](https://iudx.org.in) data discovery portal.
 The consumers can access data from the resource server proxy using HTTPs.
 
 <p align="center">
-<img src="./docs/rs_overview.png">
+<img src="./docs/RS Proxy.png">
 </p>
 
 
@@ -18,7 +18,7 @@ The consumers can access data from the resource server proxy using HTTPs.
 - Search and count APIs for searching through available data: Support for Complex (Temporal +  Attribute), Temporal (Before, during, After) and Attribute searches.
 - Integration with authorization server (token introspection) to serve private data as per the access control policies set by the provider
 - Secure data access over TLS 
-- Scalable, service mesh architecture based implementation using open source components: Vert.X API framework, Elasticsearch/Postgres for database.
+- Scalable, service mesh architecture based implementation using open source components: Vert.X API framework, Postgres for database.
 - Hazelcast and Zookeeper based cluster management and service discovery
 
 
@@ -53,14 +53,9 @@ Find the installations of the above along with the configurations to modify the
 
 ### JAR based
 1. Install java 11 and maven
-2. Set Environment variables
-```
-export RS_URL=https://<rs-domain-name> **Doubt**
-export LOG_LEVEL=INFO
-```
-3. Use maven to package the application as a JAR
+2. Use maven to package the application as a JAR
    `mvn clean package -Dmaven.test.skip=true`
-4. 2 JAR files would be generated in the `target/` directory
+3. 2 JAR files would be generated in the `target/` directory
    - `iudx.rs.proxy-cluster-0.0.1-SNAPSHOT-fat.jar` - clustered vert.x containing micrometer metrics
    - `iudx.rs.proxy-dev-0.0.1-SNAPSHOT-fat.jar` - non-clustered vert.x and does not contain micrometer metrics
 

docs/openapi.yaml

@@ -14,7 +14,7 @@ consumes:
 
 info:
   title: "Data exchange Resource Server Proxy APIs"
-  version: 4.0.0
+  version: 4.5.0
   description: |
     # Introduction
     The Resource Proxy Server is DXs'(Data Exchange) data discovery portal.
@@ -236,9 +236,9 @@ paths:
         - lang: 'cURL'
           label: 'get encrypted data'
           source: |
-            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/entities?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/pune-env-flood/FWR055&q=referenceLevel>15.0' \
-            --header 'token: <tokenValue>'\
-            --header 'publicKey: <publicKeyValues>'
+            curl --location -g --request GET 'https://example-proxy.com/ngsi-ld/v1/entities?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/surat-itms-realtime-information/surat-itms-live-eta&geoproperty=location&georel=near;maxdistance=10&geometry=Point&coordinates=[21.178,72.834]&offset=0&limit=10' \
+            --header 'token: <tokenValue>' \
+            --header 'publicKey: <publicKeyValue>'
 
         - lang: 'cURL'
           label: 'search by circle'
@@ -970,9 +970,16 @@ paths:
         - lang: 'cURL'
           label: 'get data read query'
           source: |
-            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/consumer/audit?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/pune-env-flood/FWR055&timerel=during&time=2022-03-20T14:20:00Z&endTime=2022-03-24T14:20:00Z&api=/ngsi-ld/v1/temporal/entities' \
+            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/consumer/audit?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/pune-env-flood/FWR055&timerel=during&time=2022-11-20T14:20:00Z&endTime=2022-11-21T14:20:00Z&api=/ngsi-ld/v1/temporal/entities' \
             --header 'token: <tokenValue>'
 
+        - lang: 'cURL'
+          label: 'get data count query'
+          source: |
+            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/consumer/audit?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/surat-itms-realtime-information/surat-itms-live-eta&timerel=during&time=2022-11-20T14:20:00Z&endTime=2022-11-21T14:20:00Z&api=/ngsi-ld/v1/entities' \
+            --header 'token: <tokenValue>' \
+            --header 'options: count'
+
       responses:
         200:
           description: 'search status'
@@ -1073,9 +1080,16 @@ paths:
         - lang: 'cURL'
           label: 'get data read query'
           source: |
-            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/provider/audit?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/surat-itms-realtime-information/surat-itms-live-eta&timerel=during&time=2021-11-20T14:20:00Z&endTime=2021-12-02T14:20:00Z&api=/ngsi-ld/v1/entityOperations/query&providerID=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86' \
+            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/provider/audit?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/surat-itms-realtime-information/surat-itms-live-eta&timerel=during&time=2022-11-20T14:20:00Z&endTime=2022-11-21T14:20:00Z&api=/ngsi-ld/v1/entityOperations/query&providerID=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86' \
             --header 'token: <tokenValue>'
 
+        - lang: 'cURL'
+          label: 'get data count query'
+          source: |
+            curl --location --request GET 'https://example-proxy.com/ngsi-ld/v1/provider/audit?id=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/example-proxy.com/surat-itms-realtime-information/surat-itms-live-eta&timerel=during&time=2022-11-20T14:20:00Z&endTime=2022-11-21T14:20:00Z&api=/ngsi-ld/v1/entityOperations/query&providerID=iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86' \
+            --header 'token: <tokenValue>' \
+            --header 'options: count'
+
       responses:
         200:
           description: 'search status'

examples/consumers(adapter)/example-secrets/secrets/config.ini

@@ -0,0 +1,13 @@
+[server_setup]
+username = <rs-proxy-adapter-user>
+password = <rs-proxy-adapter-user passwd>
+host = <databroker-host>
+port = 24567
+vhost = IUDX-INTERNAL
+
+
+[collection_queue]
+queue = <queue-name>
+
+
+

examples/consumers(adapter)/rabbitmq.py → examples/consumers(adapter)/src/rabbitmq.py

@@ -2,14 +2,25 @@
 import ssl
 import json
 import base64
+import logging
+from configparser import ConfigParser
 from nacl.public import SealedBox, PublicKey
 
 
-username = ''
-password = ''
-host = 'databroker.iudx.io'
-port = 24567
-vhost = 'IUDX-INTERNAL'
+
+config = ConfigParser(interpolation=None)
+config.read("secrets/config.ini")
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+time_format = "%Y-%m-%dT%H:%M:%SZ"
+time_formatter = "%Y-%m-%d"
+
+
+username = str(config.get('server_setup', 'username'))
+password = str(config.get('server_setup', 'password'))
+host = str(config.get('server_setup', 'host'))
+port = config.get('server_setup', 'port')
+vhost = str(config.get('server_setup', 'vhost'))
 context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
 
 ssl_options=pika.SSLOptions(context)
@@ -31,7 +42,7 @@ def on_message_received(ch, method, properties, body):
 
     # Get the JSON message from the body
 
-    json_data = open(r"data_txt.json", "w")
+    json_data = open(r"../data_txt.json", "w")
     json_data.write(body.decode())
     json_data.close()
     # convert properties to dictionary format
@@ -47,7 +58,7 @@ def on_message_received(ch, method, properties, body):
         requestJson = json.loads(body)
         requestId = requestJson['searchType'];
         dictionary = {'adapter': 'pune-aqm', 'correlation_id': properties.correlation_id, 'reply_to_queue': properties.reply_to,
-                      'id': 'requestId', 'response': requestJson}
+                      'id': 'requestId', 'response': requestJson, 'statusCode': 200}
         jsonString = json.dumps(dictionary, indent=4)
         reply_queue = properties.reply_to
         print(" [INFO] Publishing query response")
@@ -70,7 +81,7 @@ def on_message_received(ch, method, properties, body):
 
         # Load the JSON data to a python variable
 
-        with open('data_txt.json') as f:
+        with open('../data_txt.json') as f:
             message = json.load(f)
 
         message_bytes = json.dumps(message).encode()
@@ -93,7 +104,7 @@ def on_message_received(ch, method, properties, body):
         }
         requestJson = json.loads(body)
         requestId = requestJson['searchType'];
-        dictionary = {'adapter':'pune-aqm','correlation_id':properties.correlation_id, 'reply_to_queue':properties.reply_to, 'id':requestId,'response':requestJson,'results': {'encryptedData':str_encrypted}}
+        dictionary = {'adapter':'pune-aqm', 'correlation_id':properties.correlation_id, 'reply_to_queue':properties.reply_to, 'id':requestId,'response':requestJson,'results': {'encryptedData':str_encrypted}, 'statusCode':200}
         jsonString = json.dumps(dictionary, indent=4)
 
 

examples/consumers(adapter)/src/requirements.txt

@@ -0,0 +1,6 @@
+configparser==5.3.0
+pika==1.3.1
+requests==2.28.2
+PyNaCl==1.5.0
+urllib3==1.26.14
+configparser==5.3.0

examples/data_txt.json

@@ -0,0 +1 @@
+{"id":["iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information/surat-itms-live-eta"],"geo-query":{"lat":21.178,"lon":72.834,"radius":10.0,"geoproperty":"location"},"offset":"0","limit":"10","searchType":"latestSearch_geoSearch","instanceID":"localhost:8090","applicableFilters":["SPATIAL","TEMPORAL","ATTR"],"publicKey":"1uxmO5GqpqootKRxK_6f_HvJ7ownd_ejwd_kZBN-bzM="}

src/main/java/iudx/rs/proxy/apiserver/ApiServerVerticle.java

@@ -345,16 +345,16 @@ public void handleTemporalQuery(RoutingContext routingContext) {
    */
   private void adapterResponseForCountQuery(RoutingContext context, JsonObject json,
                                             HttpServerResponse response) {
-//    json.put("publicKey", "Some_value");
-
     String publicKey = context.request().getHeader(HEADER_PUBLIC_KEY);
     json.put(HEADER_PUBLIC_KEY, publicKey);
     brokerService.executeAdapterQueryRPC(json, handler -> {
       if (handler.succeeded()) {
         LOGGER.info("Success: Count Success");
-        JsonObject adapterResponse=handler.result();
+        JsonObject adapterResponse = handler.result();
+        adapterResponse.put("type", ResponseUrn.SUCCESS_URN.getUrn());
+        adapterResponse.put("title", ResponseUrn.SUCCESS_URN.getMessage());
         response.putHeader(CONTENT_TYPE, APPLICATION_JSON)
-                .setStatusCode(adapterResponse.getInteger("status"))
+                .setStatusCode(adapterResponse.getInteger("statusCode"))
                 .end(adapterResponse.toString());
       } else {
         LOGGER.error("Fail: Count Fail");
@@ -378,14 +378,14 @@ private void adapterResponseForSearchQuery(RoutingContext context, JsonObject js
     brokerService.executeAdapterQueryRPC(json, handler -> {
       if (handler.succeeded()) {
         JsonObject adapterResponse=handler.result();
-        int status=adapterResponse.containsKey("status")?adapterResponse.getInteger("status"):400;
+        int status=adapterResponse.containsKey("statusCode")?adapterResponse.getInteger("statusCode"):400;
+        System.out.println("adapter response status code : " + adapterResponse.getInteger("statusCode"));
         response.putHeader(CONTENT_TYPE, APPLICATION_JSON);
         response.setStatusCode(status);
         if(status==200) {
           LOGGER.info("Success: adapter call Success with {}",status);
           adapterResponse.put("type", ResponseUrn.SUCCESS_URN.getUrn());
           adapterResponse.put("title", ResponseUrn.SUCCESS_URN.getMessage());
-          adapterResponse.remove("status");
           response.end(adapterResponse.toString());
           context.data().put(RESPONSE_SIZE, response.bytesWritten());
           Future.future(fu -> updateAuditTable(context));
@@ -398,7 +398,7 @@ private void adapterResponseForSearchQuery(RoutingContext context, JsonObject js
         }
 
       } else {
-        LOGGER.error("Fail: Search Fail");
+        LOGGER.error("Failure: Adapter Search Fail");
         response.putHeader(CONTENT_TYPE, APPLICATION_JSON)
                 .setStatusCode(400)
                 .end(handler.cause().getMessage());

src/main/java/iudx/rs/proxy/apiserver/util/ApiServerConstants.java

@@ -116,7 +116,7 @@ public class ApiServerConstants {
       Pattern.compile(
           "^[a-zA-Z0-9.-]{4,100}/{1}[a-zA-Z0-9.]{4,100}/{1}[a-zA-Z.-]{4,100}/{1}[a-zA-Z-_.]{4,100}/{1}[a-zA-Z0-9-_.]{4,100}$");
   public static final int VALIDATION_MAX_ATTRS = 5;
-  public static final int VALIDATION_MAX_DAYS_INTERVAL_ALLOWED = 31;
+  public static final int VALIDATION_MAX_DAYS_INTERVAL_ALLOWED = 10;
   public static final int VALIDATION_COORDINATE_PRECISION_ALLOWED = 6;
   public static final int VALIDATIONS_MAX_ATTR_LENGTH = 100;
   public static final int VALIDATION_ALLOWED_COORDINATES = 10;

src/main/java/iudx/rs/proxy/apiserver/validation/ValidatorsHandlersFactory.java

@@ -99,7 +99,7 @@ private List<Validator> getTemporalRequestValidations(
     validators.add(new GeoRelTypeValidator(parameters.get(NGSILDQUERY_GEOREL), false));
     validators.add(new GeometryTypeValidator(parameters.get(NGSILDQUERY_GEOMETRY), false));
     validators.add(new GeoPropertyTypeValidator(parameters.get(NGSILDQUERY_GEOPROPERTY), false));
-    validators.add(new QTypeValidator(parameters.get(NGSILDQUERY_OPERATOR), false));
+    validators.add(new QTypeValidator(parameters.get(NGSLILDQUERY_Q), false));
     validators.add(new DistanceTypeValidator(parameters.get(NGSILDQUERY_MAXDISTANCE), false));
     validators.add(new DistanceTypeValidator(parameters.get("maxDistance"), false));
     validators.add(new OptionsTypeValidator(parameters.get(IUDXQUERY_OPTIONS), false));

src/main/java/iudx/rs/proxy/authenticator/JwtAuthenticationServiceImpl.java

@@ -218,9 +218,8 @@ public Future<JsonObject> validateAccess(JwtData jwtData, boolean openResource,
     AuthorizationRequest authRequest = new AuthorizationRequest(method, api);
     IudxRole role = IudxRole.fromRole(jwtData.getRole());
     AuthorizationStrategy authStrategy = AuthorizationContextFactory.create(role,apis);
-    LOGGER.info("strategy : " + authStrategy.getClass().getSimpleName());
     JwtAuthorization jwtAuthStrategy = new JwtAuthorization(authStrategy);
-    LOGGER.info("auth strategy " + jwtAuthStrategy);
+    LOGGER.info("auth strategy " + authStrategy.getClass().getSimpleName());
     LOGGER.info("endPoint : " + authInfo.getString("apiEndpoint"));
     if (jwtAuthStrategy.isAuthorized(authRequest, jwtData)) {
       LOGGER.info("User access is allowed.");

src/main/java/iudx/rs/proxy/authenticator/authorization/AuthorizationContextFactory.java

@@ -12,6 +12,9 @@ public static AuthorizationStrategy create(IudxRole role,Api apis) {
       case PROVIDER: {
         return ProviderAuthStrategy.getInstance(apis);
       }
+      case DELEGATE: {
+        return DelegateAuthStrategy.getInstance(apis);
+      }
       default:
         throw new IllegalArgumentException(role + "role is not defined in IUDX");
     }

src/main/java/iudx/rs/proxy/authenticator/authorization/DelegateAuthStrategy.java

@@ -0,0 +1,48 @@
+package iudx.rs.proxy.authenticator.authorization;
+
+import iudx.rs.proxy.authenticator.model.JwtData;
+import iudx.rs.proxy.common.Api;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class DelegateAuthStrategy implements AuthorizationStrategy {
+
+  private static final Logger LOGGER = LogManager.getLogger(DelegateAuthStrategy.class);
+
+  static Map<String, List<AuthorizationRequest>> providerAuthorizationRules = new HashMap<>();
+  private final Api apis;
+  private static volatile DelegateAuthStrategy instance;
+
+  private DelegateAuthStrategy(Api apis) {
+    this.apis=apis;
+    buildPermissions(apis);
+  }
+
+  public static DelegateAuthStrategy getInstance(Api api)
+  {
+    if (instance == null)
+    {
+      synchronized (DelegateAuthStrategy.class)
+      {
+        if (instance == null)
+        {
+          instance = new DelegateAuthStrategy(api);
+        }
+      }
+    }
+    return instance;
+  }
+  private void buildPermissions(Api api) {
+    // delegate allowed to access all endpoints
+
+  }
+
+  @Override
+  public boolean isAuthorized(AuthorizationRequest authRequest, JwtData jwtData) {
+    return true;
+  }
+}

src/main/java/iudx/rs/proxy/authenticator/authorization/IudxRole.java

@@ -5,6 +5,7 @@
 public enum IudxRole {
 
   CONSUMER("consumer"),
+  DELEGATE("delegate"),
   PROVIDER("provider");
 
   private final String role;

src/test/java/iudx/rs/proxy/apiserver/query/QueryMapperTest.java

@@ -186,8 +186,7 @@ public void testToJson4SimpleAttributeQuery(Vertx vertx, VertxTestContext testCo
         assertTrue(json.containsKey(NGSILDQUERY_ID));
         assertTrue(json.containsKey(NGSILDQUERY_ATTRIBUTE));
         assertTrue(json.containsKey(JSON_ATTR_QUERY));
-        assertTrue(json.getJsonArray(JSON_ATTR_QUERY) instanceof JsonArray);
-        assertEquals(json.getJsonArray(JSON_ATTR_QUERY).size(), 2);
+        assertTrue(json.getString(JSON_ATTR_QUERY) != null);
         testContext.completeNow();
     }