From c2920e0951faf4e8406800f4e4bb33be941ba966 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 20 Sep 2023 12:16:31 -0700 Subject: [PATCH 01/17] first version without considering vector --- docker-compose/docker-compose-dev-mode.yml | 47 ++--- nosqlbench/hdrdata.log | 41 ++++ nosqlbench/hdrstats.log | 5 + nosqlbench/http-jsonapi-vector-crud.yaml | 35 ++-- pom.xml | 5 + .../FilterClauseDeserializer.java | 12 +- .../operation/model/impl/DBFilterBase.java | 81 +++++++- .../operation/model/impl/FindOperation.java | 193 +++++++++++++++--- .../service/processor/CommandProcessor.java | 3 + .../processor/MeteredCommandProcessor.java | 2 + .../model/impl/FindOneCommandResolver.java | 14 ++ .../model/impl/matcher/FilterMatchRules.java | 21 ++ .../model/impl/matcher/FilterMatcher.java | 2 + .../impl/matcher/FilterableResolver.java | 33 ++- .../service/shredding/model/AtomicValue.java | 2 + .../shredding/model/DocValueHasher.java | 2 + 16 files changed, 421 insertions(+), 77 deletions(-) create mode 100644 nosqlbench/hdrdata.log create mode 100644 nosqlbench/hdrstats.log diff --git a/docker-compose/docker-compose-dev-mode.yml b/docker-compose/docker-compose-dev-mode.yml index 7f19e92d5c..25fedd8617 100644 --- a/docker-compose/docker-compose-dev-mode.yml +++ b/docker-compose/docker-compose-dev-mode.yml @@ -8,6 +8,7 @@ services: ports: - "9042:9042" - "8081:8081" + - "8091:8091" mem_limit: 2G environment: - JAVA_OPTS="-Xmx1536M" @@ -22,29 +23,29 @@ services: timeout: 10s retries: 10 - jsonapi: - image: ${JSONIMAGE}:${JSONTAG} - depends_on: - coordinator: - condition: service_healthy - networks: - - stargate - ports: - - "8181:8181" - mem_limit: 2G - environment: - - JAVA_MAX_MEM_RATIO=75 - - JAVA_INITIAL_MEM_RATIO=50 - - GC_CONTAINER_OPTIONS=-XX:+UseG1GC - - QUARKUS_GRPC_CLIENTS_BRIDGE_HOST=coordinator - - QUARKUS_GRPC_CLIENTS_BRIDGE_PORT=8091 - - QUARKUS_HTTP_ACCESS_LOG_ENABLED=${REQUESTLOG} - - QUARKUS_LOG_LEVEL=${LOGLEVEL} - healthcheck: - test: curl -f http://localhost:8181/stargate/health || exit 1 - interval: 5s - timeout: 10s - retries: 10 +# jsonapi: +# image: ${JSONIMAGE}:${JSONTAG} +# depends_on: +# coordinator: +# condition: service_healthy +# networks: +# - stargate +# ports: +# - "8181:8181" +# mem_limit: 2G +# environment: +# - JAVA_MAX_MEM_RATIO=75 +# - JAVA_INITIAL_MEM_RATIO=50 +# - GC_CONTAINER_OPTIONS=-XX:+UseG1GC +# - QUARKUS_GRPC_CLIENTS_BRIDGE_HOST=coordinator +# - QUARKUS_GRPC_CLIENTS_BRIDGE_PORT=8091 +# - QUARKUS_HTTP_ACCESS_LOG_ENABLED=${REQUESTLOG} +# - QUARKUS_LOG_LEVEL=${LOGLEVEL} +# healthcheck: +# test: curl -f http://localhost:8181/stargate/health || exit 1 +# interval: 5s +# timeout: 10s +# retries: 10 networks: stargate: diff --git a/nosqlbench/hdrdata.log b/nosqlbench/hdrdata.log new file mode 100644 index 0000000000..392cabba73 --- /dev/null +++ b/nosqlbench/hdrdata.log @@ -0,0 +1,41 @@ +#logging histograms for session nFEb91a +#[Histogram log format version 1.3] +#[StartTime: 1695055647.732 (seconds since epoch), Mon Sep 18 09:47:27 PDT 2023] +"StartTimestamp","Interval_Length","Interval_Max","Interval_Compressed_Histogram" +Tag=bind,1.012,28.996,3.940,HISTFAAAACd42pNpmSzMwMDAwQABLFCaEUza//9v/wEi8HcPJ9P/xRxMAIofCDA= +Tag=execute,1.016,29.003,5262.541,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4jA3zXKTJd7mJkAq40Jsg== +Tag=tries,1.019,29.015,0.000,HISTFAAAAB142pNpmSzMwMDAxAABLFCaEUoz2X+AyQAAP4gCgw== +Tag=result,1.021,29.014,5272.764,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4jA/zXKTPN7mZkAqtAJgQ== +Tag=result_success,1.023,29.027,5272.764,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4jA/zXKTPN7mZkAqtAJgQ== +Tag=read_input,1.030,29.036,0.620,HISTFAAAACd42pNpmSzMwMDAwQABLFCaEUzy//9v/wEi8Pc2G9PdS6xMAIdVCCY= +Tag=strides_servicetime,1.033,29.039,7113.540,HISTFAAAACF42pNpmSzMwMDAwgABMJoRhP8Dgf0HiMDf5epMAITCCEk= +Tag=cycles_servicetime,1.036,29.052,5305.008,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4hA+1plpt5+ZiYAptUI+g== +Tag=statuscode,1.048,29.055,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAY/CwBKGQOL +Tag=bind,8.309,21.795,6.689,HISTFAAAA/J42j1VTWgdVRTOfO/kvhdr2g6juIhZTmfhqhZa8fLQTRBxMcxSXMrQpbh8vIUEDaGU8ighhBDKo4QsJAQpIiIupDwkSCkiUiSISBFXpahICeIift+588zk3bn3nPN95+eee+flT7aKhYX+1YX0Z9078/Gjs7M3/kyCp/d7eLRs3wR8m+Pz5+z3Pm4vo+LzZBE1vs/wdR/7Pm6cwwPgIKN4jJEmlO8Z7vSwHSh8Bhz3cBf4w1DiNHMW6knUUJDsTzXatEfRw0xQjn/1iDUqiU0j5Y5vSGsjzTWj4m+gMULGVN7puVTvEWcVRmGMKaSzRpaVyCtRPtCiVkCWSCnTT3E17oeLSiE0NCAHalKM3E3l7DV/TShVkFEibvAd7WgrTvdSyc/US/RwTkcHlHs5DjKfdXJ3KW+l+0gklZOnUB3qrJWVKaDScSp/EypjSFZ32QpA/lGYo8gTKnrt8lW1rbJrVtA8HxRomQtyonI+hd5UcU5Q454KkzY6DXeMSzdQdKWZ7ITLaSVLqZo5B2jBBUOTfTKqrO2UhTUDvYdoHdy6VaS2RPIQHSNk2yFGnKdQpI1CM55kk5z7PGiMgesQFU5C8WlD5eyVM9JO/jzBxlKqBUvZut/GY4umBBV1blqJmRlYySoWHvkQ7/suReEHRddJMSiT2O2mF8Ujqn0j8iRPBbNUmVSptAF1YrPUptHxUXF5rF4v5W+p5K3XYth1UC4p8xY2qk9LqnKFQ7qopuvqr4gwDl0UHs3YR+VU07y21I3OCAemYqpE7LI5y7xr2xC7HEpcp9HQIxe+Yj+OzCGtl32SUae2/R9eht+knuhMVMYjNXR+rjXxcyZe4kXcdj3rHauejjZOTazO9udWQqYDxNYx324iRSak6qITkZpKVtftJvCVLjavaZ46kPWW4STztpHoV3AlW8+o6ZpL5R+qTK3JzRdIdxt3xpSpArnX04Y8lpDQiY4nr4qoS+qZLqxIP/9kgv6su5AsugMOHFIZ78exR04ar5Nja6wHj39/4MrpIj4grM9HDbCqGN7DjR7W+GJyM3nfgZx8SJmnewt4qkZZwiPoOiP7fZDgxGQ2yzzHNVzGWzhSWY90885UkHdFtQ67jEPQ10/S7mbY9jquiog33syUw8zv+YiTTNnpW2K4bTKN8v7lov6J2h3IbvM89hcxOYdNgk3sdLQVcLfEL8s4W8Jm3w4vYO9FzFaw9xJ2Ax4XOORnKMePFU5ewL3n8Q42gj5Km6/j41ewm2PrKj67gn8DNi7h0yWsX8PxeUxXsHMRZ4adV/GEcc3exFEPx8uYXMIPr+F4FW/j9CJurmC7j801bBS4ghtLrPV/djUh9g== +Tag=execute,8.310,21.802,5636.096,HISTFAAABe942i2WD2RWexjH39/3d87d3t5eM7P7mt15zcx0J5lkZmaSa5JJJsnMNcnMJJNcc+Wa7mS6MzNJkkwyycxMMpMkSSaTyZXkSiZJupMkSffzPd39e8/5nd/veZ7vn+c5++nP2ZpcLt+W+/6V/P8Z/PuNr+5/vy88/6ter6MuFXU/aCpqRlqUVqIeVmo66KNU1kLQ7DY9yOtRorsFrTToJlvyOhv0d9DnoBvSlrQeNBH0NPrQlVTTlarVRtAprUcVtRz0JdWmNBv0JNV4lW5u06AWErXredDVqElpPK970p0fWNtiTZpP9bsqtK5kKUt2NtFlEXcl1YdEjbqb6LR2uZ530vuiHgW9jGpTr55zL53dprlKjQdidGouusZriZapMdGtoJN6lHfAPXoTNKBzQWc0H9RtFFqQbgfVGOCgHgQdUosTnZOO6BexdjtytRLUqj5Fh9uMehd1NShVzk+vSDtU0l49T3RcF0kprUWzdD3A63qqMT0MehwpoKxhnY16E7XqZxMCQ47v6ai5RI+DZRk2hK+BWNy//wFWbzlDrat7G9WlZwXqzVuA1UqNwGfUqNk9zke9/gkqGNU+l7hsmvp1SaCaCzoIlFO6ETSfqEnXxMJOPUxA8kEC4YuAJjtYrSLDzaDz2U+1jnHqpFYjIpR0Ql8caUSfpAPcsWMfELagJahBvxraDsL3wl+Bo82I1w7bzZoMuhxsqKhxcaSHaI0cPmYIF4O+StOJNgMoYK9HF6QlK4KZdsv0ZaoNgb0DYUb0OuhCsClrWX6TGbPLXisSFXAdVF7w4oUEDo5oKpCJNAd0Leionph8rvao3b7rxj0UiLdI9yEQpGQGZ5y705iatJ/zbWAZTU7IVt9LLbiuVi9NwZpsrTqed5AfDSuIMMyBIaMFIR1WcsJeF1SGyyF2D+hjYHsfC9MONS/bvUiuAZutk7pLLNe5c7u9JeVYhe7IMr6KWgz270UbG3Y2HaLH4DMXN5G95MXJ6CbdiIQ4CPHINGGpYW+XVdulw5ZlwVbt0UyqS8HAyLYYaZYq5zoE3IItfRiJlwy+bE93WdJbFn4+JGMUHrHUCFVWcdXompqo5RQll9GtwMMr1veGXboW9FAOOeFmGIOYBoDCwKpINy1bdhQCImw04alZo/P6ECFpqSkbjWGRN/FTIgu+vpwiO52z7jRbmZprIeFpjYDSx8PdSN6q3+A2byW2LFojrLRSXDMITrCeZ++lkFQAo9Uzq85kFKiyE+xJF7y+yPgxpAKLFhef75ApaGN7n0GV3IURShc9P44Qf8ye2nKoyax++r0NsnoBlrdEhwjS5fZO6J4W08TijHsTjnMwn5lkVH+Ytn0QPWyvVhNk3Id3w2ALvdFIolE9s39GSL8uolWzst/KwcgRmmA5C1xt7yBJR7KTtWxyDNgL+B5Sr9kqX6Kz1rjyCSXcuRlHPLwGWTUpzfbYnEXsQ5slrJ8w1kt0XR0lt0LiF3dcs3V5Jk8lLj7LQ/dTQrBJt1sdHnlfQWk05qq7lFbs1wr9wHCQXmd22EgNvD8b/t+C53InMh/3y+k4qZ9lk7PFgtywrPsxPGyf5nIvKfZBUU7vjR+9Xtu1RejssHnOhwxOq9PWm5mjKBLdXI1uh0FQV7FzzgPjDPxvhAzYqtuyqHMeT8ueFe0slx0bfHnyQVMR91L1CH2WJ11/kvPIamWhqHuO14DmewlWy358W8XNcPa3Kuubbj57eLgzqWaDe6gnm55lyE05XHLNNSTJUWaev2006R4ONeq6sw8TvpdbBDhFmAqW+tmVcpVmj3N2cz83uznepvqkHyeXlUsOYIbIyGjhWARaA6uNtosbv8C563blgv0ZuT9qOj5HltZsSsZlP5jve+7aq/Ugue53R5lQ835dL2Wji2NpRvYYOnHygShlwIPlvCUyxLf8x5J6XlVbkcWQvV+6aOKjXDExXvhfkPbv75anQv9xD6oqv/l4IRKet9yrvF9hVyoobz3RZKU31ml5u8ZTzW33QHn3s6Z+1PxB/Qf7kp7D +Tag=tries,8.314,21.814,0.000,HISTFAAAAB942pNpmSzMwMDAzAABLFCaEUoz2X+Asi7wAwBD1gNf +Tag=result,8.315,21.813,5636.096,HISTFAAABfR42i2WD0TdaxjHz/v9/c5OZy1J0k03x5FJN5NjdzJJMkkmM5mZJMlkZiYzV+a60p0r3WRmkiuTJJlJMkmumZlJJjNzZWauSa6Z7szMZHY/31+3v+e8v/d9nuf753ne8/2vt8tTqeyPqYOv+P//wb/f+Gr992Dh7e/V+jvSdImeBo1HmpSWpD8jPS3SZNAHqVb3gm4d1kZWT2I9LtZajRaktaxGgv4K+hJ0T974LOgmK5E+SnfTmihShV4EXdPzSMVaCfqa1o50K+hFWiOlWjysfi3FatKroJlIY9JoVo+l9UM6oY+sSQtp3VBWW4o5T7DR2KsVWk3rU6ycHsa6rkYXsMdPiTaC3kQqqEuvkppGDmu+SL8orWbNRdoKmo31gBL5GzRkUNOigt2gPv0W9LPmg1r1MlJK96XVoDID7Dc7Z1TnRJR5Tu0873egC97ToG5FDrgT6X2ku4F8Ka1GrrWBatv0Ktag7sRalTYi0wSnNdpMa9iRn0UAzumSRiO9i7QWeHNTJCTKZKS5WJvBslzSbKSvQRcN5P0hleiBVE+CJ8HnWrRdrHK4/gSHRezWSARG2B3UVVXrTeAZqE65wpVIx9WjKYFq1uDOw+RC0EKM4nNO3qinMUBgERCvAyzVs1pGhkWskvyUaQB9r2o9Ynelrmg/QM6gCzjNO3acAsIHWDHcPq0HYtSiTjuVDOgoKZug+6jGgv5A3QCNo+TrIFieswNmZCpoX5qMteMg7Olw1UtBZ3U7ACIysFayFjvGZWt5lSTtFqsc7ncTX7bofgxheWNrVimbWUSRHiSkzgGnOa25wNvnokJenVBTvOzQGe9Apzs0RCBIlWaCfXycSECqVSfnC0AZiq/ITm+jGrqjQn+bgU0ZXyXPT5IfCTNEGOQAmxKN7vspCbtcUA4qB9nQ5xbI46y8JhxqkbY0gk712mrNCFbJd5UeOR9bMvgio4eyim8jE7SWlNzrhR2H6NC8IPuem7rA2Z2k5Z/TWREozyBVCpvctNSca7RqBfQEyZKd2q7bad0nckSfrJv/NoM9R6mfgvZsQAR4YJUWQjxMrWlNuxfKiJunDrOeAWAeFwwlr7JsOuZTSdo8z5ss6lTSIPOJcDkmwGlCVrv4dZGVGTXr5j2P9J1AqXe15eSYEMzVWakJG41hkXVTs3zNvp5JUwFTasvpEJJEmyGesEsA1E2Zx6GwFiKaOTgVa9xsNUDEnNmbVpwhgIsuBwynq7BINTU38Jvj0HKIW2CMXmF0ZlksVkuMuNi8XuajwJFug6p0E0YIsuTpcR4Oht2UH9yhY0n5tHsB6rpInPXk6wZDC4mzMTrUuQVZvOXWhPAUEky6xiGLNWfTXab/P7pDC146C7Qcx/rg+AbtsW0DXSb/lghXzkon1ZvHc5C9kkQuswwY/WR8jLVkcvS6/TA+pNJGNdqPnLbccowq3vcYSxGW2dWfDKIc/GGyWavYjTbL9r6nYiV9Z4s3QOi+e67OI25bHkvPkn7/HPQ5RuCxg4bLaC8DT7Tmuvt0w/Oa4fcw8izYTQzBrYJjepLp/y0xfzPqXHTjXCT3djI661g0eVUAbjXf13nZRopTcJTSXiAJiu3axCUQetLuGQ8JnganrTY1F9Ak8uTP2+D9kFvKzlk3zg1umxchAbbu2VGiMU/aFU+LJpZz+gdkIkC3R10Jfpv31GpkpYLqU77eGgjzyAOoBs3bCFVBdWdZLCVtM8qWJqvuoFb+d1LcsbiMHSUc7SBGB0/6sE8NTG9Yodrkwsry1x19gkN591MJsVrgq5QHtTisIrGtq0gTPZ08T9nQPbwt8LCg6rgHN+eUik/jhyi5FivY6mLd2jMe4i1ESRlYco02yJ8ALpiQL75ZN+1LRmYPqB979tqu1UCZ90WUg6cFj7PlZBBwLJ3QPYxSnHwikvV6ZoxbJGN8Jw+oO46KJsuWpYsSfuJsrT+rvPankKaD6+Wlx/+IfV7qq5+rg/BcdG+zvsVmMpS3FWusyBurtHJEI2nNHvFMef+DFr/TeJf+A099nbw= +Tag=result_success,8.315,21.830,5636.096,HISTFAAABfR42i2WD0TdaxjHz/v9/c5OZy1J0k03x5FJN5NjdzJJMkkmM5mZJMlkZiYzV+a60p0r3WRmkiuTJJlJMkmumZlJJjNzZWauSa6Z7szMZHY/31+3v+e8v/d9nuf753ne8/2vt8tTqeyPqYOv+P//wb/f+Gr992Dh7e/V+jvSdImeBo1HmpSWpD8jPS3SZNAHqVb3gm4d1kZWT2I9LtZajRaktaxGgv4K+hJ0T974LOgmK5E+SnfTmihShV4EXdPzSMVaCfqa1o50K+hFWiOlWjysfi3FatKroJlIY9JoVo+l9UM6oY+sSQtp3VBWW4o5T7DR2KsVWk3rU6ycHsa6rkYXsMdPiTaC3kQqqEuvkppGDmu+SL8orWbNRdoKmo31gBL5GzRkUNOigt2gPv0W9LPmg1r1MlJK96XVoDID7Dc7Z1TnRJR5Tu0873egC97ToG5FDrgT6X2ku4F8Ka1GrrWBatv0Ktag7sRalTYi0wSnNdpMa9iRn0UAzumSRiO9i7QWeHNTJCTKZKS5WJvBslzSbKSvQRcN5P0hleiBVE+CJ8HnWrRdrHK4/gSHRezWSARG2B3UVVXrTeAZqE65wpVIx9WjKYFq1uDOw+RC0EKM4nNO3qinMUBgERCvAyzVs1pGhkWskvyUaQB9r2o9Ynelrmg/QM6gCzjNO3acAsIHWDHcPq0HYtSiTjuVDOgoKZug+6jGgv5A3QCNo+TrIFieswNmZCpoX5qMteMg7Olw1UtBZ3U7ACIysFayFjvGZWt5lSTtFqsc7ncTX7bofgxheWNrVimbWUSRHiSkzgGnOa25wNvnokJenVBTvOzQGe9Apzs0RCBIlWaCfXycSECqVSfnC0AZiq/ITm+jGrqjQn+bgU0ZXyXPT5IfCTNEGOQAmxKN7vspCbtcUA4qB9nQ5xbI46y8JhxqkbY0gk712mrNCFbJd5UeOR9bMvgio4eyim8jE7SWlNzrhR2H6NC8IPuem7rA2Z2k5Z/TWREozyBVCpvctNSca7RqBfQEyZKd2q7bad0nckSfrJv/NoM9R6mfgvZsQAR4YJUWQjxMrWlNuxfKiJunDrOeAWAeFwwlr7JsOuZTSdo8z5ss6lTSIPOJcDkmwGlCVrv4dZGVGTXr5j2P9J1AqXe15eSYEMzVWakJG41hkXVTs3zNvp5JUwFTasvpEJJEmyGesEsA1E2Zx6GwFiKaOTgVa9xsNUDEnNmbVpwhgIsuBwynq7BINTU38Jvj0HKIW2CMXmF0ZlksVkuMuNi8XuajwJFug6p0E0YIsuTpcR4Oht2UH9yhY0n5tHsB6rpInPXk6wZDC4mzMTrUuQVZvOXWhPAUEky6xiGLNWfTXab/P7pDC146C7Qcx/rg+AbtsW0DXSb/lghXzkon1ZvHc5C9kkQuswwY/WR8jLVkcvS6/TA+pNJGNdqPnLbccowq3vcYSxGW2dWfDKIc/GGyWavYjTbL9r6nYiV9Z4s3QOi+e67OI25bHkvPkn7/HPQ5RuCxg4bLaC8DT7Tmuvt0w/Oa4fcw8izYTQzBrYJjepLp/y0xfzPqXHTjXCT3djI661g0eVUAbjXf13nZRopTcJTSXiAJiu3axCUQetLuGQ8JnganrTY1F9Ak8uTP2+D9kFvKzlk3zg1umxchAbbu2VGiMU/aFU+LJpZz+gdkIkC3R10Jfpv31GpkpYLqU77eGgjzyAOoBs3bCFVBdWdZLCVtM8qWJqvuoFb+d1LcsbiMHSUc7SBGB0/6sE8NTG9Yodrkwsry1x19gkN591MJsVrgq5QHtTisIrGtq0gTPZ08T9nQPbwt8LCg6rgHN+eUik/jhyi5FivY6mLd2jMe4i1ESRlYco02yJ8ALpiQL75ZN+1LRmYPqB979tqu1UCZ90WUg6cFj7PlZBBwLJ3QPYxSnHwikvV6ZoxbJGN8Jw+oO46KJsuWpYsSfuJsrT+rvPankKaD6+Wlx/+IfV7qq5+rg/BcdG+zvsVmMpS3FWusyBurtHJEI2nNHvFMef+DFr/TeJf+A099nbw= +Tag=read_input,8.328,21.833,0.065,HISTFAAAAQx42j1QsUpDQRB8N7fvPF+iBGxtrPwGq3zCsvhLSrC2EIuQ4pHS2kKsrMTawkIsxCqNQeRhFWfvQu4tzN7O7Lzhji+uj5omzJp6ZIuhwmYz/a7dQ8Y8YgjZkkK0U4GJTrwfQw9Vsuas5JLmRExQsTFpg0+zdlBY5i0TO3EQTZb7IG6Jntb0MjFHXyMv1KY+oCBFjLAMZHwJX3tS9rycUefIKBZ+MX5sngH3wiMT4QVVU1viIrq9yToKcdgWzUkXP0YsXsvgglWLJ/HxPJaEJtXG/P9/ZTwUfwp3ZT64aUsgxTriYx+3Ld5G3tf4rNcD3Gfc8TFxjp+JL/6eyecUl6dYnWA2wvsV8A+IWlao +Tag=strides_servicetime,8.329,21.835,10710.680,HISTFAAAAZh42i2RwStEURTG5/zuu+81mIaEhCRkIWsLyUKSLCQLSbKSLP0BktCkSa9pkqxkIVlYSJIsNM3CQtM0SZpkYWGhaUKykMS5ceveW1/nnu93vtu8lq6NRIhG/pb3f4tu86Nr4PVPyIcd7HTxHePMJxFjr5oyZKsYIOuzIpzHnVDLewU5KAmWZMBVBadCJ98BJcuLoVjPLA1EufZ4gFXIBKQMO54rf4ZlHoRJ6tgUMtBNCc4M98KmZU8IhUNRn1XhAippJSPOsYu08CmMqvLhcRowrW1y+szTqtBnF/qIkbZcCuPUUBbeIK+nMONcj2GOD2EDWvg09KtND2O0sUTBY5gJApLCjejrNtr5QQcJPabo5d2yDYdwIqxbxejmUbgTCIErwajpELeQRBHnGOTcUAiYdyBNHAmXuInnFXKGIuzj8FVPCFuw4BA16zgHaKJRDiwFnw2rSesQT+rECI0OoYdrSMCXS1KB4/pLi2qqNKFhHVJQjCpPzlezbVe1G5DXPsY5pSy3VZSNK054TsmKpqmfnAr4BT9Bc+M= +Tag=cycles_servicetime,8.330,21.850,5636.358,HISTFAAABel42i2WAWSVexjGz//5vq+zrzqOyXGkm5nJ7M7MZJKZzByTmUwymckkyXVdVzJJrtlNMnNkMkkyRzJJkkkmM8lkkkmSJDPJlXRz5Zpcu7/nq/M5Z9/3//7v+z7v8z7v+99Pf07vyOXSA7nvn/jH3+DvJp8Df39f+DK1S1cjTRZ0N6ga6Yq0JK1EelSnhaBV6YD+0Gaqd6nexKpt09pu/RdUSzUT9CDoa9C89E16FHQx6HWkD9Jcost12qmVoBN6H2mb7gRtJlqTbgYtJ5or6tJWHdaDWAf1JGg60mVpPNUTaXGLWv10gc0JPuM3Ul5Xgt4FbUSqSWf0KlY1MWQMx4N26640EelrQZ+Cbkdq1pCe4TBSdatT6VejOjQbGdF0rPlIE4mWgob1KlVVvFsPOqbJQLI13z2MlNMtaTko1UensRqgosWB4KhfPdLloAsRLvDToKPSfNDnSGuRbgTZ/FGkqyKXsipaj0F0NdY9aTXSpWByt+lJonOGtBSpC4QnncPHSA+d0aR0SHJNarFhYPmLagAXIaHi7RYVdF9qU72eBpt16/U2kLRpRXpYp9OA+CT9rrdBp3gq2SqvRWmvc1w0SyMmutWAhzWgMd0KuhXjo+bg7VqJldgH/LwOmNkgJd79YNbJv17HQTVmQlvBfMZyGOHakHr1q6pBfdzMRfocyG/IVO2BxIMsFmC5GVj7oXWPGbmROW3Rq0DQfu3AoIIrKEFpVHIq1oegXboQNKhLcvUHjaETSG8caERFXc8UF3QtWIe74ISHCy7djN/MiT2NumFZF7gdz+QwCrJJv54OsDDrFJ6LOxB1kbXuBfwU7IY64Ocf167s14TvYM+iszuEWaf2wQeZ26aXgmJTNIRePZVjlNiwj82zbooSdelw1bMGIp+y22PAkJq5TtFBJ/Tlu7p2GlzRklyMwdIH6AzWEeqdg3rar8d856lIqoXM3dvIe2hSmmfUrgGyA/KpbrPuCL+dhFwLmor0Iuh5hItByE8JNBHAiV3F9WgnDAt3LNdezSRusfsR5SbqYYzm/eeYi08DHNDVYIU0OuJZ0tzhxNHZbcs65XoW9DL492m2eB4gK363lHX8dVsNsK2iJuQwbdJymjKrTRbQkgGCxlnvRuPjQkO0TDNqbGS1DTmlLk4ODwn1OwJfA2xt5XVkDuq5BRtds1P/BkdYDjH9VorPs7k+q0kLJl2Yz8TWWhMW7e6akls6Dz9tbtCSGXfPdZtoYHS4x6ohy7KAPXMvHsPXOd5FRBv0jCzD2LAHTkK0CmsFrlZ8tqLDNpKHAapecFNMBU+cDowPES3vMh2mabohNaUv+0A2l7EJvy2uQ4QKq44xRtBayNrwpN05LVg7BCFNXKN8xxgOryyiUzhC8RXLtd9VL/tnCKngkfFYb9XD6P64nbWC22j0xwTAomYlMZmrVhe7GNz6Frlxzuqbh2kp46jJ7TvrOh+1eCOl8Q1TOAwvx7L82XyUMqIFRj4bUAh9scEhE5PXpCkpw8GXPNuAu2Cwy5Y2JxDjbEZ6L0dYTTzIRkh4QJvB2thP1OO6LfrphM+oTc8hyjPngdKHZGnp0+DoAUUFq5wH1pBPrb+8owDMbhf8kqdEGdsVcwTnIxQxcYs1WPXHCVkk2E0Py3Nm6IUnWtFQe/By0azcD9mYaMHkQ5ZgigQmPJiabHgSH3mcDwMCPq2HJffibhD0ZkNj0GdREYencNlDcfKZAutJpASPIOziNsfrY9DWgK8hfA7C7CjPPpnq8ZNg1sra3u+BxdIveBngkfxOY5Xg3iiS7Cpj7aNkJOupFI87fRL1KxcnABvOJNrDy06gNuCl3kPY4yDP702frHdNRkKgo67JhpGsWDMIsYK/xy5lBRSuAwX8jZ3NLlHe3TVjcTxOsMlBbfa/xWO34ojDTIZsFDToo/R4S5YhNbnnITTgu0ZQNXl4vrPbLleGkSfyzU6TorXDHFxItBxrnfkZ6ZphP4s1WaeXIvv57RpPNLvd/658/lnrDVqt6H8PWaFD +Tag=statuscode,8.331,21.865,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AQBKhQP3 +Tag=statuscode,8.331,21.865,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AQBKhQP3 +Tag=statuscode,8.337,21.859,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HABKcwPl +Tag=statuscode,8.338,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/FABKbwPh +Tag=statuscode,8.338,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/GABKawPd +Tag=statuscode,8.338,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/DABKcQPj +Tag=statuscode,8.338,21.859,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HgBKgwP1 +Tag=statuscode,8.338,21.859,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/FABKbwPh +Tag=statuscode,8.339,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HgBKgwP1 +Tag=statuscode,8.339,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AQBKhQP3 +Tag=statuscode,8.339,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AABKZQPX +Tag=statuscode,8.340,21.857,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/BgBKfQPv +Tag=statuscode,8.341,21.856,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/CgBKeQPr +Tag=statuscode,8.341,21.856,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HgBKgwP1 +Tag=statuscode,8.342,21.855,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/BQBKjQP/ +Tag=statuscode,8.343,21.854,0.000,HISTFAAAAB142pNpmSzMwMDAzIAKGKH0f/sPEAZ/AyMATsEECQ== +Tag=statuscode,8.345,21.852,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/CgBKeQPr +Tag=statuscode,8.348,21.850,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AgBKdQPn +Tag=statuscode,8.349,21.849,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAa/FwBKXwPR +Tag=statuscode,8.354,21.844,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AABKZQPX diff --git a/nosqlbench/hdrstats.log b/nosqlbench/hdrstats.log new file mode 100644 index 0000000000..2a46985098 --- /dev/null +++ b/nosqlbench/hdrstats.log @@ -0,0 +1,5 @@ +#logging stats for session nFEa4dy +#[Histogram log format version 1.0] +#[StartTime: 1695055365.611 (seconds since epoch), Mon Sep 18 09:42:45 PDT 2023] +#[TimeUnit: NANOSECONDS] +#Tag,Interval_Start,Interval_Length,count,min,p25,p50,p75,p90,p95,p98,p99,p999,p9999,max diff --git a/nosqlbench/http-jsonapi-vector-crud.yaml b/nosqlbench/http-jsonapi-vector-crud.yaml index 54767930f8..b94a61e106 100644 --- a/nosqlbench/http-jsonapi-vector-crud.yaml +++ b/nosqlbench/http-jsonapi-vector-crud.yaml @@ -3,6 +3,7 @@ min_version: "5.17.3" # Example command line # Against AstraDB # nb5 -v http-jsonapi-vector-crud docscount=1000 threads=20 jsonapi_host=Your-AstraDB-Host auth_token=Your-AstraDB-Token jsonapi_port=443 protocol=https path_prefix=/api/json namespace=Your-Keyspace +# java -jar ~/nb5.jar -v http-jsonapi-vector-crud docscount=10000 threads=100 jsonapi_host=cc6478dd-2d0e-4576-87f1-fa3cbd3e10bb auth_token=AstraCS:OssyPlglELzWhYaIJgdqluCN:c136258f28ab5348f6288d06b0e794c72b8c8d89e163a62b0ab5a3bff61a522b-us-east-2.apps.astra-dev.datastax.com jsonapi_port=443 protocol=https path_prefix=/api/json namespace=photography # Against local JSON API # nb5 -v http-jsonapi-vector-crud jsonapi_host=localhost docscount=1000 threads=20 @@ -16,9 +17,9 @@ scenarios: default: schema: run driver=http tags==block:schema threads==1 cycles==UNDEF write: run driver=http tags==name:"write.*" cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn - read: run driver=http tags==name:"read.*" cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn - update: run driver=http tags==name:"update.*" cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn - delete: run driver=http tags==name:"delete.*" cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn +# read: run driver=http tags==name:"read.*" cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn +# update: run driver=http tags==name:"update.*" cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn +# delete: run driver=http tags==name:"delete.*" cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn bindings: # To enable an optional weighted set of hosts in place of a load balancer @@ -44,20 +45,20 @@ bindings: blocks: schema: ops: - create-namespace: - method: POST - uri: <>://{weighted_hosts}:<><>/v1 - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "{token}" - Content-Type: "application/json" - ok-body: ".*\"ok\":1.*" - body: >2 - { - "createNamespace": { - "name": "<>" - } - } +# create-namespace: +# method: POST +# uri: <>://{weighted_hosts}:<><>/v1 +# Accept: "application/json" +# X-Cassandra-Request-Id: "{request_id}" +# X-Cassandra-Token: "{token}" +# Content-Type: "application/json" +# ok-body: ".*\"ok\":1.*" +# body: >2 +# { +# "createNamespace": { +# "name": "<>" +# } +# } delete-collection: method: POST diff --git a/pom.xml b/pom.xml index abbf640bd1..431d00c572 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,11 @@ org.apache.commons commons-lang3 + + com.bpodgursky + jbool_expressions + 1.23 + io.quarkus quarkus-junit5 diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java index 6a7e853f4b..ebfd7dab9d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.quarkus.logging.Log; import io.smallrye.config.SmallRyeConfig; import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.ArrayComparisonOperator; import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.ComparisonExpression; @@ -65,7 +66,7 @@ public FilterClause deserialize( entry.getKey(), jsonNodeValue(entry.getKey(), entry.getValue()))); } } - + Log.info("deserialize expressionList" + expressionList); validate(expressionList); return new FilterClause(expressionList); } @@ -90,10 +91,11 @@ private void validate(String path, FilterOperation filterOperation) { if (filterOperation.operator() instanceof ValueComparisonOperator valueComparisonOperator) { switch (valueComparisonOperator) { case IN -> { - if (!path.equals(DocumentConstants.Fields.DOC_ID)) { - throw new JsonApiException( - ErrorCode.INVALID_FILTER_EXPRESSION, "Can use $in operator only on _id field"); - } + // if (!path.equals(DocumentConstants.Fields.DOC_ID)) { + // throw new JsonApiException( + // ErrorCode.INVALID_FILTER_EXPRESSION, "Can use $in operator only on _id + // field"); + // } if (filterOperation.operand().value() instanceof List list) { if (list.size() > operationsConfig.defaultPageSize()) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java index d4fb6a141d..2d2daf0037 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java @@ -16,10 +16,7 @@ import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId; import io.stargate.sgv2.jsonapi.util.JsonUtil; import java.math.BigDecimal; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -288,13 +285,85 @@ boolean canAddField() { } } } + + /** + * Filters db documents based on non document id field values if id use "$in" operator, use + * IDFilter, since partition key 'or' is not supported besides id, "$in" operator, use INFilter + */ + public static class INFilter extends DBFilterBase { + private final Object arrayValue; + protected final INFilter.Operator operator; + + @Override + JsonNode asJson(JsonNodeFactory nodeFactory) { + return DBFilterBase.getJsonNode(nodeFactory, arrayValue); + } + + @Override + boolean canAddField() { + return false; + } + + public enum Operator { + IN; + } + + public INFilter(INFilter.Operator operator, String path, Object arrayValue) { + super(path); + this.arrayValue = arrayValue; + this.operator = operator; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + INFilter inFilter = (INFilter) o; + return operator == inFilter.operator && Objects.equals(arrayValue, inFilter.arrayValue); + } + + @Override + public int hashCode() { + return Objects.hash(arrayValue, operator); + } + + private static final String DATA_CONTAINS = "array_contains"; + + @Override + public BuiltCondition get() { + // For IN filter we always use getALL() method + return null; + } + + public List getAll() { + List values = (List) arrayValue; + switch (operator) { + case IN: + if (values.isEmpty()) return List.of(); + return values.stream() + .map( + v -> + BuiltCondition.of( + DATA_CONTAINS, + Predicate.CONTAINS, + getGrpcValue(getHashValue(new DocValueHasher(), getPath(), v)))) + .collect(Collectors.toList()); + + default: + throw new JsonApiException( + ErrorCode.UNSUPPORTED_FILTER_OPERATION, + String.format("Unsupported %s column operation %s", getPath(), operator)); + } + } + } + /** * DB filter / condition for testing a set value Note: we can only do CONTAINS until SAI indexes - * are updated + * are updated Now, cassandra supports or operation, we can add $in */ public abstract static class SetFilterBase extends DBFilterBase { public enum Operator { - CONTAINS; + CONTAINS, } protected final String columnName; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index d302bc8fa0..d63030f183 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -1,9 +1,14 @@ package io.stargate.sgv2.jsonapi.service.operation.model.impl; +import com.bpodgursky.jbool_expressions.And; +import com.bpodgursky.jbool_expressions.Expression; +import com.bpodgursky.jbool_expressions.Or; +import com.bpodgursky.jbool_expressions.Variable; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import io.quarkus.logging.Log; import io.smallrye.mutiny.Uni; import io.stargate.bridge.proto.QueryOuterClass; import io.stargate.sgv2.api.common.cql.builder.BuiltCondition; @@ -23,6 +28,7 @@ import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -277,6 +283,9 @@ public static FindOperation sorted( @Override public Uni> execute(QueryExecutor queryExecutor) { + Log.info("give me a executor " + commandContext); +// Log.info("give me a executor! " + filters.get(0)); + final boolean vectorEnabled = commandContext().isVectorEnabled(); if (vector() != null && !vectorEnabled) { return Uni.createFrom() @@ -325,6 +334,7 @@ public Uni getDocuments( } case DOCUMENT, KEY -> { List queries = buildSelectQueries(additionalIdFilter); + Log.error("getDocuments query before findDocument " + queries); return findDocument( queryExecutor, queries, @@ -381,34 +391,89 @@ public ReadDocument getNewDocument() { * buildConditions method. */ private List buildSelectQueries(DBFilterBase.IDFilter additionalIdFilter) { - List> conditions = buildConditions(additionalIdFilter); - if (conditions == null) { - return List.of(); - } - List queries = new ArrayList<>(conditions.size()); - conditions.forEach( - condition -> { - if (vector() == null) { - queries.add( - new QueryBuilder() - .select() - .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) - .from(commandContext.namespace(), commandContext.collection()) - .where(condition) - .limit(limit) - .build()); - } else { - QueryOuterClass.Query builtQuery = getVectorSearchQuery(condition); - final List valuesList = - builtQuery.getValuesOrBuilder().getValuesList(); - final QueryOuterClass.Values.Builder builder = QueryOuterClass.Values.newBuilder(); - valuesList.forEach(builder::addValues); - builder.addValues(CustomValueSerializers.getVectorValue(vector())); - queries.add(QueryOuterClass.Query.newBuilder(builtQuery).setValues(builder).build()); + AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); + // Log.error("filter is " + filters); + // Log.error("filter is " + filters.size()); + // Log.error("filter is " + (filters.get(0) instanceof DBFilterBase.INFilter)); + // Log.error("filter is " + (filters.get(1) instanceof DBFilterBase.INFilter)); + + filters.forEach( + filter -> { + if (filter instanceof DBFilterBase.INFilter) { + hasInFilterBesidesIdField.set(true); + return; // need to break } }); - return queries; + if (hasInFilterBesidesIdField.get()) { + Log.info("laile lao tie"); + List> expressions = buildConditionExpressions(additionalIdFilter); + Log.info(expressions.size()); + Log.info(expressions); + + if (expressions == null) { + return List.of(); + } + List queries = new ArrayList<>(expressions.size()); + expressions.forEach( + expression -> { + if (vector() == null) { + Log.error("each expression " + expression); + queries.add( + new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .build()); + } else { + // TODO vector search part , compatible with or operation? + Log.warn("pre arrive "); + QueryOuterClass.Query builtQuery = getVectorSearchQuery(null); + final List valuesList = + builtQuery.getValuesOrBuilder().getValuesList(); + final QueryOuterClass.Values.Builder builder = QueryOuterClass.Values.newBuilder(); + valuesList.forEach(builder::addValues); + builder.addValues(CustomValueSerializers.getVectorValue(vector())); + queries.add(QueryOuterClass.Query.newBuilder(builtQuery).setValues(builder).build()); + } + }); + + return queries; + + } else { + Log.error("not has expression"); + + List> conditions = buildConditions(additionalIdFilter); + if (conditions == null) { + return List.of(); + } + List queries = new ArrayList<>(conditions.size()); + conditions.forEach( + condition -> { + if (vector() == null) { + queries.add( + new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .from(commandContext.namespace(), commandContext.collection()) + .where(condition) + .limit(limit) + .build()); + } else { + QueryOuterClass.Query builtQuery = getVectorSearchQuery(condition); + final List valuesList = + builtQuery.getValuesOrBuilder().getValuesList(); + final QueryOuterClass.Values.Builder builder = QueryOuterClass.Values.newBuilder(); + valuesList.forEach(builder::addValues); + builder.addValues(CustomValueSerializers.getVectorValue(vector())); + queries.add(QueryOuterClass.Query.newBuilder(builtQuery).setValues(builder).build()); + } + }); + + return queries; + } } /** Making it a separate method to build vector search query as there are many options */ @@ -462,6 +527,8 @@ private QueryOuterClass.Query getVectorSearchQuery(List conditio } } } else { + + Log.warn("arrive " + conditions); return new QueryBuilder() .select() .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) @@ -518,6 +585,7 @@ private List buildSortedSelectQueries( */ private List> buildConditions(DBFilterBase.IDFilter additionalIdFilter) { List conditions = new ArrayList<>(filters.size()); + Log.error("lkk " + filters.size()); DBFilterBase.IDFilter idFilterToUse = additionalIdFilter; // if we have id filter overwrite ignore existing IDFilter boolean idFilterOverwrite = additionalIdFilter != null; @@ -525,6 +593,9 @@ private List> buildConditions(DBFilterBase.IDFilter additio if (!(filter instanceof DBFilterBase.IDFilter idFilter)) { conditions.add(filter.get()); } else { + // final DBFilterBase.IDFilter filter1 = (DBFilterBase.IDFilter) filter; + // Log.error("jiujiu " + filter1.values); + if (!idFilterOverwrite) { idFilterToUse = idFilter; } @@ -552,6 +623,78 @@ private List> buildConditions(DBFilterBase.IDFilter additio } } + /** + * Builds select query based on filters and additionalIdFilter overrides. return expression to + * pass logic operation information, eg 'or' + */ + private List> buildConditionExpressions( + DBFilterBase.IDFilter additionalIdFilter) { + Expression conditionExpression = null; + // for (DBFilterBase filter : filters) { + // // all filters will be in And.of + // // if the filter is DBFilterBase.INFilter + // // inside of this filter, it has getAll method to return a list of BuiltCondition, + // these + // // BuiltCondition will be in Or.of + // } + DBFilterBase.IDFilter idFilterToUse = additionalIdFilter; + // if we have id filter overwrite ignore existing IDFilter + boolean idFilterOverwrite = additionalIdFilter != null; + for (DBFilterBase filter : filters) { + if (filter instanceof DBFilterBase.INFilter inFilter) { + List conditions = inFilter.getAll(); + Log.info(" size " + conditions.size()); + Log.info(" conditions " + conditions); + if (!conditions.isEmpty()) { + List> variableConditions = + conditions.stream().map(Variable::of).toList(); + Log.info(" size" + variableConditions.size()); + + conditionExpression = + conditionExpression == null + ? Or.of(variableConditions) + : And.of(Or.of(variableConditions), conditionExpression); + + Log.info(" conditionExpression " + conditionExpression); + } + } else if (filter instanceof DBFilterBase.IDFilter idFilter) { + if (!idFilterOverwrite) { + idFilterToUse = idFilter; + } + } else { + conditionExpression = + conditionExpression == null + ? Variable.of(filter.get()) + : And.of(Variable.of(filter.get()), conditionExpression); + } + } + + Log.info(filters.size()); + Log.info(filters); + Log.info(conditionExpression); + if (idFilterToUse != null) { + final List inSplit = idFilterToUse.getAll(); + if (inSplit.isEmpty()) { + return null; + } else { + // split n queries by id + Expression tempExpression = conditionExpression; + return inSplit.stream() + .map( + idCondition -> { + Expression newExpression = + tempExpression == null + ? Variable.of(idCondition) + : And.of(Variable.of((idCondition)), tempExpression); + return newExpression; + }) + .collect(Collectors.toList()); + } + } else { + return conditionExpression == null ? null : List.of(conditionExpression); // only one query + } + } + /** * Represents sort field name and option to be sorted ascending/descending. * diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java index d2515ae8c5..aaf4176521 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.service.processor; +import io.quarkus.logging.Log; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; @@ -58,7 +59,9 @@ public Uni processCommand( .flatMap( resolver -> { // if we have resolver, resolve operation and execute + Log.info("begin to resolve command! "); Operation operation = resolver.resolveCommand(commandContext, command); + // Log.info("give me a resolved operation " + operation); return operation.execute(queryExecutor); }) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index ed74434d8b..53900e4705 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -4,6 +4,7 @@ import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Timer; +import io.quarkus.logging.Log; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.api.common.StargateRequestInfo; import io.stargate.sgv2.api.common.config.MetricsConfig; @@ -75,6 +76,7 @@ public Uni processCommand( CommandContext commandContext, T command) { Timer.Sample sample = Timer.start(meterRegistry); // start by resolving the command, get resolver + Log.info("command :" + command); return commandProcessor .processCommand(commandContext, command) .onItem() diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java index 7d33206011..53112f5b68 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl; import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortClause; import io.stargate.sgv2.jsonapi.api.model.command.impl.FindOneCommand; @@ -37,7 +38,19 @@ public Class getCommandClass() { @Override public Operation resolveCommand(CommandContext commandContext, FindOneCommand command) { + Log.info("??????!"); + List filters = resolve(commandContext, command); + Log.info("filters after resolve !!!" + filters); + Log.info("filters after resolve !!!" + filters.size()); + + // for (DBFilterBase filter : filters) { + // Log.info("filter after resolve lhs columnName~~~~ " + filter.get().lhs().columnName()); + // Log.info("filter after resolve lhs value ~~~~ " + filter.get().lhs().value()); + // Log.info("filter after resolve predicate ~~~~ " + filter.get().predicate()); + // Log.info("filter after resolve filter value ~~~~ " + filter.get().value().toString()); + // + // } final SortClause sortClause = command.sortClause(); float[] vector = SortClauseUtil.resolveVsearch(sortClause); @@ -74,6 +87,7 @@ public Operation resolveCommand(CommandContext commandContext, FindOneCommand co // documentConfig.defaultPageSize() as limit operationsConfig.maxDocumentSortCount()); } else { + Log.error("uns 11"); return FindOperation.unsortedSingle( commandContext, filters, command.buildProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java index 4deda3eec1..8068b5d40e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl.matcher; import com.google.common.annotations.VisibleForTesting; +import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.Filterable; @@ -59,6 +60,26 @@ public FilterMatchRule addMatchRule( * @return */ public List apply(CommandContext commandContext, T command) { + Log.error("begin to apply " + command); + Log.error("matchRules " + matchRules); + Log.error("matchRules len " + matchRules.size()); + + final long count1 = + matchRules.stream() + .map(e -> e.apply(commandContext, command)) + .filter(Optional::isPresent) + .count(); + Log.error("dqdw " + count1); + final long count = + matchRules.stream() + .map(e -> e.apply(commandContext, command)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + .orElseThrow() + .size(); + Log.error("woLLL " + count); + return matchRules.stream() .map(e -> e.apply(commandContext, command)) .filter(Optional::isPresent) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java index a4d1033218..b6524bfc7a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl.matcher; +import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.Filterable; import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.ComparisonExpression; @@ -37,6 +38,7 @@ public enum MatchStrategy { } public Optional> apply(T command) { + Log.error("onece"); FilterClause filter = command.filterClause(); List unmatchedCaptures = new ArrayList<>(captures); CaptureGroups captures = new CaptureGroups(command); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java index 668ac00514..f05fdae6a1 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl.matcher; +import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.Filterable; @@ -34,7 +35,7 @@ public abstract class FilterableResolver { private static final Object ID_GROUP = new Object(); private static final Object ID_GROUP_IN = new Object(); - + private static final Object DYNAMIC_GROUP_IN = new Object(); private static final Object DYNAMIC_TEXT_GROUP = new Object(); private static final Object DYNAMIC_NUMBER_GROUP = new Object(); private static final Object DYNAMIC_BOOL_GROUP = new Object(); @@ -48,6 +49,7 @@ public abstract class FilterableResolver { @Inject public FilterableResolver() { + Log.error("should just once"); matchRules.addMatchRule(this::findNoFilter, FilterMatcher.MatchStrategy.EMPTY); matchRules @@ -62,6 +64,12 @@ public FilterableResolver() { .capture(ID_GROUP_IN) .compareValues("_id", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); + // matchRules + // .addMatchRule(this::findDynamic, FilterMatcher.MatchStrategy.STRICT) + // .matcher() + // .capture(DYNAMIC_GROUP_IN) + // .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); + // NOTE - can only do eq ops on fields until SAI changes matchRules .addMatchRule(this::findDynamic, FilterMatcher.MatchStrategy.GREEDY) @@ -70,6 +78,8 @@ public FilterableResolver() { .compareValues("_id", EnumSet.of(ValueComparisonOperator.EQ), JsonType.DOCUMENT_ID) .capture(ID_GROUP_IN) .compareValues("_id", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY) + .capture(DYNAMIC_GROUP_IN) + .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY) .capture(DYNAMIC_NUMBER_GROUP) .compareValues("*", EnumSet.of(ValueComparisonOperator.EQ), JsonType.NUMBER) .capture(DYNAMIC_TEXT_GROUP) @@ -93,10 +103,12 @@ public FilterableResolver() { } protected List resolve(CommandContext commandContext, T command) { + Log.info("FilterableResolver resolve!"); return matchRules.apply(commandContext, command); } private List findById(CommandContext commandContext, CaptureGroups captures) { + Log.info("find id"); List filters = new ArrayList<>(); final CaptureGroup idGroup = (CaptureGroup) captures.getGroupIfPresent(ID_GROUP); @@ -122,10 +134,13 @@ private List findById(CommandContext commandContext, CaptureGroups private List findNoFilter( CommandContext commandContext, CaptureGroups captures) { + Log.error("wogaoni"); return List.of(); } private List findDynamic(CommandContext commandContext, CaptureGroups captures) { + Log.info("find Dynamic "); + List filters = new ArrayList<>(); final CaptureGroup idGroup = @@ -141,6 +156,8 @@ private List findDynamic(CommandContext commandContext, CaptureGro final CaptureGroup> idsGroup = (CaptureGroup>) captures.getGroupIfPresent(ID_GROUP_IN); if (idsGroup != null) { + Log.error("hit1"); + idsGroup.consumeAllCaptures( expression -> filters.add( @@ -148,6 +165,20 @@ private List findDynamic(CommandContext commandContext, CaptureGro DBFilterBase.IDFilter.Operator.IN, expression.value()))); } + final CaptureGroup> dynamicGroups = + (CaptureGroup>) captures.getGroupIfPresent(DYNAMIC_GROUP_IN); + if (dynamicGroups != null) { + Log.info("hit"); + dynamicGroups.consumeAllCaptures( + expression -> { + Log.info("captured " + expression); + final DocValueHasher docValueHasher = new DocValueHasher(); + filters.add( + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, expression.path(), expression.value())); + }); + } + final CaptureGroup textGroup = (CaptureGroup) captures.getGroupIfPresent(DYNAMIC_TEXT_GROUP); if (textGroup != null) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java index bffbababeb..2f8374abcc 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.service.shredding.model; +import io.quarkus.logging.Log; import java.math.BigDecimal; import java.util.Date; @@ -18,6 +19,7 @@ public record AtomicValue(DocValueType type, String typedFullValue, DocValueHash public static AtomicValue forString(String str) { // For Strings no changes needed, use default prefix+full-value + Log.info("used here"); return create(DocValueType.STRING, str); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java index 7dcb34d87b..544c3c9a27 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.exception.ErrorCode; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.util.JsonUtil; @@ -36,6 +37,7 @@ public class DocValueHasher { final AtomicValues atomics = new AtomicValues(); public DocValueHash hash(JsonNode value) { + Log.info("inside !!"); return switch (value.getNodeType()) { case ARRAY -> arrayHash((ArrayNode) value); case BOOLEAN -> booleanValue(value.booleanValue()).hash(); From 36ad355a72c0f3cc860d63fabd70a5b54f01f672 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 20 Sep 2023 12:41:58 -0700 Subject: [PATCH 02/17] expression for vectors --- .../operation/model/impl/FindOperation.java | 64 ++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index d63030f183..66a71bd32a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -430,7 +430,7 @@ private List buildSelectQueries(DBFilterBase.IDFilter add } else { // TODO vector search part , compatible with or operation? Log.warn("pre arrive "); - QueryOuterClass.Query builtQuery = getVectorSearchQuery(null); + QueryOuterClass.Query builtQuery = getVectorSearchQueryByExpression(expression); final List valuesList = builtQuery.getValuesOrBuilder().getValuesList(); final QueryOuterClass.Values.Builder builder = QueryOuterClass.Values.newBuilder(); @@ -539,7 +539,67 @@ private QueryOuterClass.Query getVectorSearchQuery(List conditio .build(); } } - + /** A separate method to build vector search query by using expression, expression can contain logic operations like 'or','and'.. */ + private QueryOuterClass.Query getVectorSearchQueryByExpression(Expression expression) { + QueryOuterClass.Query builtQuery = null; + if (projection().doIncludeSimilarityScore()) { + switch (commandContext().similarityFunction()) { + case COSINE, UNDEFINED -> { + return new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .similarityCosine( + DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, + CustomValueSerializers.getVectorValue(vector())) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); + } + case EUCLIDEAN -> { + return new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .similarityEuclidean( + DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, + CustomValueSerializers.getVectorValue(vector())) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); + } + case DOT_PRODUCT -> { + return new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .similarityDotProduct( + DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, + CustomValueSerializers.getVectorValue(vector())) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); + } + default -> { + throw new JsonApiException( + ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME, + ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME.getMessage()); + } + } + } else { + return new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); + } + } /** * Builds select query based on filters, sort fields and additionalIdFilter overrides. * From 42ca3f048c06060060f89237770929c99399e733 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 20 Sep 2023 13:45:30 -0700 Subject: [PATCH 03/17] filter match rule unit test --- .../operation/model/impl/FindOperation.java | 12 ------- .../model/impl/FindOneCommandResolver.java | 13 ------- .../model/impl/matcher/FilterMatchRules.java | 21 ++--------- .../impl/matcher/FilterableResolver.java | 8 ++--- .../impl/matcher/FilterMatchRuleTest.java | 35 +++++++++++++++++++ 5 files changed, 40 insertions(+), 49 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index 66a71bd32a..7ec9ccc4e5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -334,7 +334,6 @@ public Uni getDocuments( } case DOCUMENT, KEY -> { List queries = buildSelectQueries(additionalIdFilter); - Log.error("getDocuments query before findDocument " + queries); return findDocument( queryExecutor, queries, @@ -392,11 +391,6 @@ public ReadDocument getNewDocument() { */ private List buildSelectQueries(DBFilterBase.IDFilter additionalIdFilter) { AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); - // Log.error("filter is " + filters); - // Log.error("filter is " + filters.size()); - // Log.error("filter is " + (filters.get(0) instanceof DBFilterBase.INFilter)); - // Log.error("filter is " + (filters.get(1) instanceof DBFilterBase.INFilter)); - filters.forEach( filter -> { if (filter instanceof DBFilterBase.INFilter) { @@ -418,7 +412,6 @@ private List buildSelectQueries(DBFilterBase.IDFilter add expressions.forEach( expression -> { if (vector() == null) { - Log.error("each expression " + expression); queries.add( new QueryBuilder() .select() @@ -443,8 +436,6 @@ private List buildSelectQueries(DBFilterBase.IDFilter add return queries; } else { - Log.error("not has expression"); - List> conditions = buildConditions(additionalIdFilter); if (conditions == null) { return List.of(); @@ -653,9 +644,6 @@ private List> buildConditions(DBFilterBase.IDFilter additio if (!(filter instanceof DBFilterBase.IDFilter idFilter)) { conditions.add(filter.get()); } else { - // final DBFilterBase.IDFilter filter1 = (DBFilterBase.IDFilter) filter; - // Log.error("jiujiu " + filter1.values); - if (!idFilterOverwrite) { idFilterToUse = idFilter; } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java index 53112f5b68..302d7a834b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java @@ -38,19 +38,7 @@ public Class getCommandClass() { @Override public Operation resolveCommand(CommandContext commandContext, FindOneCommand command) { - Log.info("??????!"); - List filters = resolve(commandContext, command); - Log.info("filters after resolve !!!" + filters); - Log.info("filters after resolve !!!" + filters.size()); - - // for (DBFilterBase filter : filters) { - // Log.info("filter after resolve lhs columnName~~~~ " + filter.get().lhs().columnName()); - // Log.info("filter after resolve lhs value ~~~~ " + filter.get().lhs().value()); - // Log.info("filter after resolve predicate ~~~~ " + filter.get().predicate()); - // Log.info("filter after resolve filter value ~~~~ " + filter.get().value().toString()); - // - // } final SortClause sortClause = command.sortClause(); float[] vector = SortClauseUtil.resolveVsearch(sortClause); @@ -87,7 +75,6 @@ public Operation resolveCommand(CommandContext commandContext, FindOneCommand co // documentConfig.defaultPageSize() as limit operationsConfig.maxDocumentSortCount()); } else { - Log.error("uns 11"); return FindOperation.unsortedSingle( commandContext, filters, command.buildProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java index 8068b5d40e..56cbcc6b73 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java @@ -60,26 +60,9 @@ public FilterMatchRule addMatchRule( * @return */ public List apply(CommandContext commandContext, T command) { - Log.error("begin to apply " + command); - Log.error("matchRules " + matchRules); - Log.error("matchRules len " + matchRules.size()); - - final long count1 = - matchRules.stream() - .map(e -> e.apply(commandContext, command)) - .filter(Optional::isPresent) - .count(); - Log.error("dqdw " + count1); - final long count = - matchRules.stream() + Log.warn("check! " + matchRules.stream() .map(e -> e.apply(commandContext, command)) - .filter(Optional::isPresent) - .map(Optional::get) - .findFirst() - .orElseThrow() - .size(); - Log.error("woLLL " + count); - + .filter(Optional::isPresent).count()); return matchRules.stream() .map(e -> e.apply(commandContext, command)) .filter(Optional::isPresent) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java index f05fdae6a1..b507fc88d7 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java @@ -49,7 +49,6 @@ public abstract class FilterableResolver { @Inject public FilterableResolver() { - Log.error("should just once"); matchRules.addMatchRule(this::findNoFilter, FilterMatcher.MatchStrategy.EMPTY); matchRules @@ -134,7 +133,6 @@ private List findById(CommandContext commandContext, CaptureGroups private List findNoFilter( CommandContext commandContext, CaptureGroups captures) { - Log.error("wogaoni"); return List.of(); } @@ -146,6 +144,8 @@ private List findDynamic(CommandContext commandContext, CaptureGro final CaptureGroup idGroup = (CaptureGroup) captures.getGroupIfPresent(ID_GROUP); if (idGroup != null) { + Log.info("hit"); + idGroup.consumeAllCaptures( expression -> filters.add( @@ -156,9 +156,7 @@ private List findDynamic(CommandContext commandContext, CaptureGro final CaptureGroup> idsGroup = (CaptureGroup>) captures.getGroupIfPresent(ID_GROUP_IN); if (idsGroup != null) { - Log.error("hit1"); - - idsGroup.consumeAllCaptures( + idsGroup.consumeAllCaptures( expression -> filters.add( new DBFilterBase.IDFilter( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java index 77d556a4d3..7cdc12550d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java @@ -4,6 +4,7 @@ import static org.mockito.Mockito.*; import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.logging.Log; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import io.stargate.sgv2.common.testprofiles.NoGlobalResourcesTestProfile; @@ -66,5 +67,39 @@ public void apply() throws Exception { filterMatchRule.apply(new CommandContext("namespace", "collection"), findOneCommand); assertThat(response).isEmpty(); } + + @Test + public void testDynamicIn() throws Exception { + String json = + """ + { + "findOne": { + "filter" : {"name" : {"$in" : ["testname1", "testname2"]}} + } + } + """; + FindOneCommand findOneCommand = objectMapper.readValue(json, FindOneCommand.class); + FilterMatcher matcher = + new FilterMatcher<>(FilterMatcher.MatchStrategy.GREEDY); + +// matcher.capture("capture marker") +// .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); + + BiFunction, List> + resolveFunction = + (commandContext, captures) -> filters; + + FilterMatchRule filterMatchRule = + new FilterMatchRule(matcher, resolveFunction); + + filterMatchRule.matcher() + .capture("capture marker") + .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); + + Optional> response = + filterMatchRule.apply(new CommandContext("testNamespace", "testCollection"), findOneCommand); + assertThat(response).isPresent(); + + } } } From 0a9e1cbcc347b1685192ab7050e43d1484a7b2e8 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 21 Sep 2023 09:44:29 -0700 Subject: [PATCH 04/17] sort support with in condition and IT test --- .../operation/model/impl/FindOperation.java | 170 ++++++---- .../model/impl/FindOneCommandResolver.java | 1 - .../model/impl/matcher/FilterMatchRules.java | 9 +- .../impl/matcher/FilterableResolver.java | 2 +- .../jsonapi/api/v1/FindIntegrationTest.java | 75 ++++- .../api/v1/VectorSearchIntegrationTest.java | 33 ++ .../model/impl/FindCommandResolverTest.java | 301 ++++++++++++++++++ .../impl/matcher/FilterMatchRuleTest.java | 25 +- 8 files changed, 526 insertions(+), 90 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index 7ec9ccc4e5..74551e5041 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -284,7 +284,7 @@ public static FindOperation sorted( @Override public Uni> execute(QueryExecutor queryExecutor) { Log.info("give me a executor " + commandContext); -// Log.info("give me a executor! " + filters.get(0)); + // Log.info("give me a executor! " + filters.get(0)); final boolean vectorEnabled = commandContext().isVectorEnabled(); if (vector() != null && !vectorEnabled) { @@ -530,65 +530,69 @@ private QueryOuterClass.Query getVectorSearchQuery(List conditio .build(); } } - /** A separate method to build vector search query by using expression, expression can contain logic operations like 'or','and'.. */ - private QueryOuterClass.Query getVectorSearchQueryByExpression(Expression expression) { + /** + * A separate method to build vector search query by using expression, expression can contain + * logic operations like 'or','and'.. + */ + private QueryOuterClass.Query getVectorSearchQueryByExpression( + Expression expression) { QueryOuterClass.Query builtQuery = null; if (projection().doIncludeSimilarityScore()) { switch (commandContext().similarityFunction()) { case COSINE, UNDEFINED -> { return new QueryBuilder() - .select() - .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) - .similarityCosine( - DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, - CustomValueSerializers.getVectorValue(vector())) - .from(commandContext.namespace(), commandContext.collection()) - .where(expression) - .limit(limit) - .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) - .build(); + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .similarityCosine( + DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, + CustomValueSerializers.getVectorValue(vector())) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); } case EUCLIDEAN -> { return new QueryBuilder() - .select() - .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) - .similarityEuclidean( - DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, - CustomValueSerializers.getVectorValue(vector())) - .from(commandContext.namespace(), commandContext.collection()) - .where(expression) - .limit(limit) - .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) - .build(); + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .similarityEuclidean( + DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, + CustomValueSerializers.getVectorValue(vector())) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); } case DOT_PRODUCT -> { return new QueryBuilder() - .select() - .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) - .similarityDotProduct( - DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, - CustomValueSerializers.getVectorValue(vector())) - .from(commandContext.namespace(), commandContext.collection()) - .where(expression) - .limit(limit) - .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) - .build(); - } - default -> { - throw new JsonApiException( - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME, - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME.getMessage()); - } - } - } else { - return new QueryBuilder() .select() .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .similarityDotProduct( + DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME, + CustomValueSerializers.getVectorValue(vector())) .from(commandContext.namespace(), commandContext.collection()) .where(expression) .limit(limit) .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) .build(); + } + default -> { + throw new JsonApiException( + ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME, + ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME.getMessage()); + } + } + } else { + return new QueryBuilder() + .select() + .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(limit) + .vsearch(DocumentConstants.Fields.VECTOR_SEARCH_INDEX_COLUMN_NAME) + .build(); } } /** @@ -600,30 +604,66 @@ private QueryOuterClass.Query getVectorSearchQueryByExpression(Expression buildSortedSelectQueries( DBFilterBase.IDFilter additionalIdFilter) { - List> conditions = buildConditions(additionalIdFilter); - if (conditions == null) { - return List.of(); - } - String[] columns = sortedDataColumns; - if (orderBy() != null) { - List sortColumns = Lists.newArrayList(columns); - orderBy().forEach(order -> sortColumns.addAll(order.getOrderingColumns())); - columns = new String[sortColumns.size()]; - sortColumns.toArray(columns); + AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); + filters.forEach( + filter -> { + if (filter instanceof DBFilterBase.INFilter) { + hasInFilterBesidesIdField.set(true); + return; // need to break + } + }); + if (hasInFilterBesidesIdField.get()) { + List> expressions = buildConditionExpressions(additionalIdFilter); + if (expressions == null) { + return List.of(); + } + String[] columns = sortedDataColumns; + if (orderBy() != null) { + List sortColumns = Lists.newArrayList(columns); + orderBy().forEach(order -> sortColumns.addAll(order.getOrderingColumns())); + columns = new String[sortColumns.size()]; + sortColumns.toArray(columns); + } + final String[] columnsToAdd = columns; + List queries = new ArrayList<>(expressions.size()); + expressions.forEach( + expression -> + queries.add( + new QueryBuilder() + .select() + .column(columnsToAdd) + .from(commandContext.namespace(), commandContext.collection()) + .where(expression) + .limit(maxSortReadLimit()) + .build())); + return queries; + + } else { + List> conditions = buildConditions(additionalIdFilter); + if (conditions == null) { + return List.of(); + } + String[] columns = sortedDataColumns; + if (orderBy() != null) { + List sortColumns = Lists.newArrayList(columns); + orderBy().forEach(order -> sortColumns.addAll(order.getOrderingColumns())); + columns = new String[sortColumns.size()]; + sortColumns.toArray(columns); + } + final String[] columnsToAdd = columns; + List queries = new ArrayList<>(conditions.size()); + conditions.forEach( + condition -> + queries.add( + new QueryBuilder() + .select() + .column(columnsToAdd) + .from(commandContext.namespace(), commandContext.collection()) + .where(condition) + .limit(maxSortReadLimit()) + .build())); + return queries; } - final String[] columnsToAdd = columns; - List queries = new ArrayList<>(conditions.size()); - conditions.forEach( - condition -> - queries.add( - new QueryBuilder() - .select() - .column(columnsToAdd) - .from(commandContext.namespace(), commandContext.collection()) - .where(condition) - .limit(maxSortReadLimit()) - .build())); - return queries; } /** diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java index 302d7a834b..7d33206011 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneCommandResolver.java @@ -1,7 +1,6 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl; import com.fasterxml.jackson.databind.ObjectMapper; -import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortClause; import io.stargate.sgv2.jsonapi.api.model.command.impl.FindOneCommand; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java index 56cbcc6b73..2702133829 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java @@ -60,9 +60,12 @@ public FilterMatchRule addMatchRule( * @return */ public List apply(CommandContext commandContext, T command) { - Log.warn("check! " + matchRules.stream() - .map(e -> e.apply(commandContext, command)) - .filter(Optional::isPresent).count()); + Log.warn( + "check! " + + matchRules.stream() + .map(e -> e.apply(commandContext, command)) + .filter(Optional::isPresent) + .count()); return matchRules.stream() .map(e -> e.apply(commandContext, command)) .filter(Optional::isPresent) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java index b507fc88d7..30eb32bea6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java @@ -156,7 +156,7 @@ private List findDynamic(CommandContext commandContext, CaptureGro final CaptureGroup> idsGroup = (CaptureGroup>) captures.getGroupIfPresent(ID_GROUP_IN); if (idsGroup != null) { - idsGroup.consumeAllCaptures( + idsGroup.consumeAllCaptures( expression -> filters.add( new DBFilterBase.IDFilter( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java index 65622a53bb..ca6d7e0c32 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java @@ -387,11 +387,43 @@ public void inConditionNonIdField() { """ { "find": { - "filter" : {"non_id" : {"$in": ["a", "b", "c"]}} - } + "filter" : { + "name" : {"$in" : ["user1", "user10"]} + } + } } """; + String expected1 = + "{\"_id\":\"doc1\", \"username\":\"user1\", \"active_user\":true, \"date\" : {\"$date\": 1672531200000}}"; + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("data.documents", hasSize(1)) + .body("status", is(nullValue())) + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected1)); + } + @Test + public void inConditionNonIdFieldIdField() { + String json = + """ + { + "find": { + "filter" : { + "name" : {"$in" : ["user1", "user10"]}, + "_id" : {"$in" : ["doc1", "???"]} + } + } + } + """; + String expected1 = + "{\"_id\":\"doc1\", \"username\":\"user1\", \"active_user\":true, \"date\" : {\"$date\": 1672531200000}}"; given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) .contentType(ContentType.JSON) @@ -400,12 +432,41 @@ public void inConditionNonIdField() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("data.documents", hasSize(1)) .body("status", is(nullValue())) - .body("data", is(nullValue())) - .body("errors", is(notNullValue())) - .body("errors[1].message", is("Can use $in operator only on _id field")) - .body("errors[1].exceptionClass", is("JsonApiException")) - .body("errors[1].errorCode", is("INVALID_FILTER_EXPRESSION")); + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected1)); + } + + @Test + public void inConditionNonIdFieldIdFieldSort() { + String json = + """ + { + "find": { + "filter" : { + "name" : {"$in" : ["user1", "user10"]}, + "_id" : {"$in" : ["doc1", "???"]} + } + "sort": { + "name": -1 } + } + } + """; + String expected1 = + "{\"_id\":\"doc1\", \"username\":\"user1\", \"active_user\":true, \"date\" : {\"$date\": 1672531200000}}"; + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("data.documents", hasSize(1)) + .body("status", is(nullValue())) + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected1)); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index 575d3de031..de4c171d85 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -571,6 +571,39 @@ public void happyPathWithFilter() { .body("errors", is(nullValue())); } + @Test + @Order(3) + public void happyPathWithInFilter() { + String json = + """ + { + "find": { + "filter" : { + "_id" : {"$in" : ["1", "2"]}, + "name": {"$in" : ["Logic Layers","???"]}, + }, + "projection" : {"_id" : 1, "$vector" : 0}, + "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, + "options" : { + "limit" : 5 + } + } + } + """; + + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("data.documents[0]._id", is("2")) + .body("data.documents[0].$vector", is(nullValue())) + .body("errors", is(nullValue())); + } + @Test @Order(4) public void happyPathWithEmptyVector() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java index 35451c6fc6..620b63cc83 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java @@ -685,4 +685,305 @@ public void noFilterConditionWithProjection() throws Exception { }); } } + + @Nested + class FindCommandResolveWithINOperator { + @Mock CommandContext commandContext; + + @Test + public void NonIdIn() throws Exception { + String json = + """ + { + "find": { + "filter" : {"name" : { "$in" : ["test1", "test2"]}} + } + } + """; + + FindCommand findCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findCommand); + + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + DBFilterBase filter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.limit()).isEqualTo(Integer.MAX_VALUE); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()).isZero(); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.orderBy()).isNull(); + assertThat(find.filters()).containsOnly(filter); + }); + + final FindOperation operation1 = (FindOperation) operation; + System.out.println("geiwo " + operation1.filters()); + } + + @Test + public void NonIdIn_IdEq() throws Exception { + String json = + """ + { + "find": { + "filter" : { + "_id" : "id1", + "name" : { "$in" : ["test1", "test2"]} + } + } + } + """; + FindCommand findCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findCommand); + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + DBFilterBase inFilter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + DBFilterBase idFilter = + new DBFilterBase.IDFilter( + DBFilterBase.IDFilter.Operator.EQ, DocumentId.fromString("id1")); + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.limit()).isEqualTo(Integer.MAX_VALUE); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()).isZero(); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.orderBy()).isNull(); + assertThat(find.filters()).containsOnly(inFilter, idFilter); + }); + } + + @Test + public void NonIdIn_IdIn() throws Exception { + String json = + """ + { + "find": { + "filter" : { + "_id" : { "$in" : ["id1", "id2"]}, + "name" : { "$in" : ["test1", "test2"]} + } + } + } + """; + FindCommand findCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findCommand); + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + DBFilterBase inFilter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + DBFilterBase idFilter = + new DBFilterBase.IDFilter( + DBFilterBase.IDFilter.Operator.IN, + List.of(DocumentId.fromString("id1"), DocumentId.fromString("id2"))); + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.limit()).isEqualTo(Integer.MAX_VALUE); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()).isZero(); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.orderBy()).isNull(); + assertThat(find.filters()).containsOnly(inFilter, idFilter); + }); + } + + @Test + public void NonIdIn_VSearch() throws Exception { + String json = + """ + { + "find": { + "filter" : { + "name" : { "$in" : ["test1", "test2"]} + }, + "sort" : {"$vector" : [0.15, 0.1, 0.1]} + } + } + """; + + FindCommand findCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findCommand); + + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + DBFilterBase inFilter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + float[] vector = new float[] {0.15f, 0.1f, 0.1f}; + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.limit()).isEqualTo(operationsConfig.maxVectorSearchLimit()); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()).isZero(); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.vector()).containsExactly(vector); + assertThat(find.filters()).containsOnly(inFilter); + }); + } + + @Test + public void NonIdIn_IdIn_VSearch() throws Exception { + String json = + """ + { + "find": { + "filter" : { + "_id" : { "$in" : ["id1", "id2"]}, + "name" : { "$in" : ["test1", "test2"]} + }, + "sort" : {"$vector" : [0.15, 0.1, 0.1]} + } + } + """; + + FindCommand findCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findCommand); + + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + DBFilterBase inFilter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + DBFilterBase idFilter = + new DBFilterBase.IDFilter( + DBFilterBase.IDFilter.Operator.IN, + List.of(DocumentId.fromString("id1"), DocumentId.fromString("id2"))); + float[] vector = new float[] {0.15f, 0.1f, 0.1f}; + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.limit()).isEqualTo(operationsConfig.maxVectorSearchLimit()); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()).isZero(); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.vector()).containsExactly(vector); + assertThat(find.filters()).containsOnly(inFilter, idFilter); + }); + } + + @Test + public void descending_sort_with_NonIdIn() throws Exception { + String json = + """ + { + "find": { + "sort": { + "name": -1 + }, + "filter" : { + "name" : {"$in" : ["test1", "test2"]} + } + } + } + """; + + FindCommand findOneCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findOneCommand); + + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + FindOperation.OrderBy orderBy = new FindOperation.OrderBy("name", false); + DBFilterBase inFilter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultSortPageSize()); + assertThat(find.limit()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.SORTED_DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()) + .isEqualTo(operationsConfig.maxDocumentSortCount()); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.orderBy()).containsOnly(orderBy); + assertThat(find.filters()).containsOnly(inFilter); + }); + } + + @Test + public void ascending_sort_with_NonIdIn_IdIn() throws Exception { + String json = + """ + { + "find": { + "sort": { + "name": 1 + }, + "filter" : { + "name" : {"$in" : ["test1", "test2"]}, + "_id" : {"$in" : ["id1","id2"]} + } + } + } + """; + + FindCommand findOneCommand = objectMapper.readValue(json, FindCommand.class); + Operation operation = resolver.resolveCommand(commandContext, findOneCommand); + + assertThat(operation) + .isInstanceOfSatisfying( + FindOperation.class, + find -> { + FindOperation.OrderBy orderBy = new FindOperation.OrderBy("name", true); + DBFilterBase inFilter = + new DBFilterBase.INFilter( + DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + DBFilterBase idFilter = + new DBFilterBase.IDFilter( + DBFilterBase.IDFilter.Operator.IN, + List.of(DocumentId.fromString("id1"), DocumentId.fromString("id2"))); + assertThat(find.objectMapper()).isEqualTo(objectMapper); + assertThat(find.commandContext()).isEqualTo(commandContext); + assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); + assertThat(find.pageSize()).isEqualTo(operationsConfig.defaultSortPageSize()); + assertThat(find.limit()).isEqualTo(operationsConfig.defaultPageSize()); + assertThat(find.pagingState()).isNull(); + assertThat(find.readType()).isEqualTo(ReadType.SORTED_DOCUMENT); + assertThat(find.skip()).isZero(); + assertThat(find.maxSortReadLimit()) + .isEqualTo(operationsConfig.maxDocumentSortCount()); + assertThat(find.singleResponse()).isFalse(); + assertThat(find.orderBy()).containsOnly(orderBy); + assertThat(find.filters()).containsOnly(inFilter, idFilter); + }); + } + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java index 7cdc12550d..3637d0abd7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRuleTest.java @@ -4,7 +4,6 @@ import static org.mockito.Mockito.*; import com.fasterxml.jackson.databind.ObjectMapper; -import io.quarkus.logging.Log; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import io.stargate.sgv2.common.testprofiles.NoGlobalResourcesTestProfile; @@ -71,7 +70,7 @@ public void apply() throws Exception { @Test public void testDynamicIn() throws Exception { String json = - """ + """ { "findOne": { "filter" : {"name" : {"$in" : ["testname1", "testname2"]}} @@ -80,26 +79,26 @@ public void testDynamicIn() throws Exception { """; FindOneCommand findOneCommand = objectMapper.readValue(json, FindOneCommand.class); FilterMatcher matcher = - new FilterMatcher<>(FilterMatcher.MatchStrategy.GREEDY); + new FilterMatcher<>(FilterMatcher.MatchStrategy.GREEDY); -// matcher.capture("capture marker") -// .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); + // matcher.capture("capture marker") + // .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); BiFunction, List> - resolveFunction = - (commandContext, captures) -> filters; + resolveFunction = (commandContext, captures) -> filters; FilterMatchRule filterMatchRule = - new FilterMatchRule(matcher, resolveFunction); + new FilterMatchRule(matcher, resolveFunction); - filterMatchRule.matcher() - .capture("capture marker") - .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); + filterMatchRule + .matcher() + .capture("capture marker") + .compareValues("*", EnumSet.of(ValueComparisonOperator.IN), JsonType.ARRAY); Optional> response = - filterMatchRule.apply(new CommandContext("testNamespace", "testCollection"), findOneCommand); + filterMatchRule.apply( + new CommandContext("testNamespace", "testCollection"), findOneCommand); assertThat(response).isPresent(); - } } } From dd8410d7850c69f4d553f255aba4127c31ca9b3c Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 21 Sep 2023 10:21:00 -0700 Subject: [PATCH 05/17] refine more tests --- .../FilterClauseDeserializerTest.java | 30 +++++++++--------- .../jsonapi/api/v1/FindIntegrationTest.java | 11 +++---- .../api/v1/FindOneIntegrationTest.java | 5 +-- .../api/v1/VectorSearchIntegrationTest.java | 31 +++++++++++++++++-- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializerTest.java index 3fa1e15af8..b425a75432 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializerTest.java @@ -290,7 +290,7 @@ public void mustHandleSubDocEq() throws Exception { } @Test - public void mustHandleIn() throws Exception { + public void mustHandleIdFieldIn() throws Exception { String json = """ {"_id" : {"$in": ["2", "3"]}} """; @@ -308,32 +308,34 @@ public void mustHandleIn() throws Exception { } @Test - public void mustHandleInArrayNonEmpty() throws Exception { + public void mustHandleNonIdFieldIn() throws Exception { String json = """ - {"_id" : {"$in": []}} + {"name" : {"$in": ["name1", "name2"]}} """; final ComparisonExpression expectedResult = new ComparisonExpression( - "_id", + "name", List.of( new ValueComparisonOperation( - ValueComparisonOperator.IN, new JsonLiteral(List.of(), JsonType.ARRAY)))); + ValueComparisonOperator.IN, + new JsonLiteral(List.of("name1", "name2"), JsonType.ARRAY)))); FilterClause filterClause = objectMapper.readValue(json, FilterClause.class); assertThat(filterClause.comparisonExpressions()).hasSize(1).contains(expectedResult); } @Test - public void mustHandleInIdFieldOnly() throws Exception { + public void mustHandleInArrayNonEmpty() throws Exception { String json = """ - {"name" : {"$in": ["aaa"]}} + {"_id" : {"$in": []}} """; - Throwable throwable = catchThrowable(() -> objectMapper.readValue(json, FilterClause.class)); - assertThat(throwable) - .isInstanceOf(JsonApiException.class) - .satisfies( - t -> { - assertThat(t.getMessage()).isEqualTo("Can use $in operator only on _id field"); - }); + final ComparisonExpression expectedResult = + new ComparisonExpression( + "_id", + List.of( + new ValueComparisonOperation( + ValueComparisonOperator.IN, new JsonLiteral(List.of(), JsonType.ARRAY)))); + FilterClause filterClause = objectMapper.readValue(json, FilterClause.class); + assertThat(filterClause.comparisonExpressions()).hasSize(1).contains(expectedResult); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java index ca6d7e0c32..f4cd002270 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java @@ -388,7 +388,7 @@ public void inConditionNonIdField() { { "find": { "filter" : { - "name" : {"$in" : ["user1", "user10"]} + "username" : {"$in" : ["user1", "user10"]} } } } @@ -416,7 +416,7 @@ public void inConditionNonIdFieldIdField() { { "find": { "filter" : { - "name" : {"$in" : ["user1", "user10"]}, + "username" : {"$in" : ["user1", "user10"]}, "_id" : {"$in" : ["doc1", "???"]} } } @@ -445,11 +445,10 @@ public void inConditionNonIdFieldIdFieldSort() { { "find": { "filter" : { - "name" : {"$in" : ["user1", "user10"]}, + "username" : {"$in" : ["user1", "user10"]}, "_id" : {"$in" : ["doc1", "???"]} - } - "sort": { - "name": -1 } + }, + "sort": { "username": -1 } } } """; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneIntegrationTest.java index 1ced23d030..be811883ff 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneIntegrationTest.java @@ -347,10 +347,7 @@ public void inConditionNonIdField() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("errors", is(notNullValue())) - .body("errors[1].message", is("Can use $in operator only on _id field")) - .body("errors[1].exceptionClass", is("JsonApiException")) - .body("errors[1].errorCode", is("INVALID_FILTER_EXPRESSION")); + .body("errors", is(nullValue())); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index de4c171d85..32731b9b9b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -575,12 +575,37 @@ public void happyPathWithFilter() { @Order(3) public void happyPathWithInFilter() { String json = + """ + { + "insertOne": { + "document": { + "_id": "xx", + "name": "Logic Layers", + "description": "ChatGPT integrated sneakers that talk to you", + "$vector": [0.25, 0.25, 0.25, 0.25, 0.25] + } + } + } + """; + + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.insertedIds[0]", is("xx")) + .body("data", is(nullValue())) + .body("errors", is(nullValue())); + json = """ { "find": { "filter" : { - "_id" : {"$in" : ["1", "2"]}, - "name": {"$in" : ["Logic Layers","???"]}, + "_id" : {"$in" : ["??", "xx"]}, + "name": {"$in" : ["Logic Layers","???"]} }, "projection" : {"_id" : 1, "$vector" : 0}, "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, @@ -599,7 +624,7 @@ public void happyPathWithInFilter() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.documents[0]._id", is("2")) + .body("data.documents[0]._id", is("xx")) .body("data.documents[0].$vector", is(nullValue())) .body("errors", is(nullValue())); } From 41041badd5dd795ec49440b93ee62ade825b72ab Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 21 Sep 2023 12:39:20 -0700 Subject: [PATCH 06/17] initial for review --- .../FilterClauseDeserializer.java | 2 -- .../operation/model/impl/FindOperation.java | 22 ------------------- .../service/processor/CommandProcessor.java | 3 --- .../processor/MeteredCommandProcessor.java | 2 -- .../model/impl/matcher/FilterMatchRules.java | 7 ------ .../model/impl/matcher/FilterMatcher.java | 2 -- .../impl/matcher/FilterableResolver.java | 9 -------- .../service/shredding/model/AtomicValue.java | 2 -- .../shredding/model/DocValueHasher.java | 2 -- .../model/impl/FindCommandResolverTest.java | 13 +++++------ 10 files changed, 6 insertions(+), 58 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java index ebfd7dab9d..c0c6be5907 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.quarkus.logging.Log; import io.smallrye.config.SmallRyeConfig; import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.ArrayComparisonOperator; import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.ComparisonExpression; @@ -66,7 +65,6 @@ public FilterClause deserialize( entry.getKey(), jsonNodeValue(entry.getKey(), entry.getValue()))); } } - Log.info("deserialize expressionList" + expressionList); validate(expressionList); return new FilterClause(expressionList); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index 74551e5041..28d5bc4491 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import io.quarkus.logging.Log; import io.smallrye.mutiny.Uni; import io.stargate.bridge.proto.QueryOuterClass; import io.stargate.sgv2.api.common.cql.builder.BuiltCondition; @@ -283,9 +282,6 @@ public static FindOperation sorted( @Override public Uni> execute(QueryExecutor queryExecutor) { - Log.info("give me a executor " + commandContext); - // Log.info("give me a executor! " + filters.get(0)); - final boolean vectorEnabled = commandContext().isVectorEnabled(); if (vector() != null && !vectorEnabled) { return Uni.createFrom() @@ -400,11 +396,7 @@ private List buildSelectQueries(DBFilterBase.IDFilter add }); if (hasInFilterBesidesIdField.get()) { - Log.info("laile lao tie"); List> expressions = buildConditionExpressions(additionalIdFilter); - Log.info(expressions.size()); - Log.info(expressions); - if (expressions == null) { return List.of(); } @@ -421,8 +413,6 @@ private List buildSelectQueries(DBFilterBase.IDFilter add .limit(limit) .build()); } else { - // TODO vector search part , compatible with or operation? - Log.warn("pre arrive "); QueryOuterClass.Query builtQuery = getVectorSearchQueryByExpression(expression); final List valuesList = builtQuery.getValuesOrBuilder().getValuesList(); @@ -518,8 +508,6 @@ private QueryOuterClass.Query getVectorSearchQuery(List conditio } } } else { - - Log.warn("arrive " + conditions); return new QueryBuilder() .select() .column(ReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns) @@ -676,7 +664,6 @@ private List buildSortedSelectQueries( */ private List> buildConditions(DBFilterBase.IDFilter additionalIdFilter) { List conditions = new ArrayList<>(filters.size()); - Log.error("lkk " + filters.size()); DBFilterBase.IDFilter idFilterToUse = additionalIdFilter; // if we have id filter overwrite ignore existing IDFilter boolean idFilterOverwrite = additionalIdFilter != null; @@ -731,19 +718,13 @@ private List> buildConditionExpressions( for (DBFilterBase filter : filters) { if (filter instanceof DBFilterBase.INFilter inFilter) { List conditions = inFilter.getAll(); - Log.info(" size " + conditions.size()); - Log.info(" conditions " + conditions); if (!conditions.isEmpty()) { List> variableConditions = conditions.stream().map(Variable::of).toList(); - Log.info(" size" + variableConditions.size()); - conditionExpression = conditionExpression == null ? Or.of(variableConditions) : And.of(Or.of(variableConditions), conditionExpression); - - Log.info(" conditionExpression " + conditionExpression); } } else if (filter instanceof DBFilterBase.IDFilter idFilter) { if (!idFilterOverwrite) { @@ -757,9 +738,6 @@ private List> buildConditionExpressions( } } - Log.info(filters.size()); - Log.info(filters); - Log.info(conditionExpression); if (idFilterToUse != null) { final List inSplit = idFilterToUse.getAll(); if (inSplit.isEmpty()) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java index aaf4176521..d2515ae8c5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/CommandProcessor.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.service.processor; -import io.quarkus.logging.Log; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; @@ -59,9 +58,7 @@ public Uni processCommand( .flatMap( resolver -> { // if we have resolver, resolve operation and execute - Log.info("begin to resolve command! "); Operation operation = resolver.resolveCommand(commandContext, command); - // Log.info("give me a resolved operation " + operation); return operation.execute(queryExecutor); }) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java index 53900e4705..ed74434d8b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/processor/MeteredCommandProcessor.java @@ -4,7 +4,6 @@ import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Timer; -import io.quarkus.logging.Log; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.api.common.StargateRequestInfo; import io.stargate.sgv2.api.common.config.MetricsConfig; @@ -76,7 +75,6 @@ public Uni processCommand( CommandContext commandContext, T command) { Timer.Sample sample = Timer.start(meterRegistry); // start by resolving the command, get resolver - Log.info("command :" + command); return commandProcessor .processCommand(commandContext, command) .onItem() diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java index 2702133829..4deda3eec1 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatchRules.java @@ -1,7 +1,6 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl.matcher; import com.google.common.annotations.VisibleForTesting; -import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.Filterable; @@ -60,12 +59,6 @@ public FilterMatchRule addMatchRule( * @return */ public List apply(CommandContext commandContext, T command) { - Log.warn( - "check! " - + matchRules.stream() - .map(e -> e.apply(commandContext, command)) - .filter(Optional::isPresent) - .count()); return matchRules.stream() .map(e -> e.apply(commandContext, command)) .filter(Optional::isPresent) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java index b6524bfc7a..a4d1033218 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterMatcher.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl.matcher; -import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.Filterable; import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.ComparisonExpression; @@ -38,7 +37,6 @@ public enum MatchStrategy { } public Optional> apply(T command) { - Log.error("onece"); FilterClause filter = command.filterClause(); List unmatchedCaptures = new ArrayList<>(captures); CaptureGroups captures = new CaptureGroups(command); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java index 30eb32bea6..c9a758b593 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl.matcher; -import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.api.model.command.Command; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.Filterable; @@ -102,12 +101,10 @@ public FilterableResolver() { } protected List resolve(CommandContext commandContext, T command) { - Log.info("FilterableResolver resolve!"); return matchRules.apply(commandContext, command); } private List findById(CommandContext commandContext, CaptureGroups captures) { - Log.info("find id"); List filters = new ArrayList<>(); final CaptureGroup idGroup = (CaptureGroup) captures.getGroupIfPresent(ID_GROUP); @@ -137,15 +134,11 @@ private List findNoFilter( } private List findDynamic(CommandContext commandContext, CaptureGroups captures) { - Log.info("find Dynamic "); - List filters = new ArrayList<>(); final CaptureGroup idGroup = (CaptureGroup) captures.getGroupIfPresent(ID_GROUP); if (idGroup != null) { - Log.info("hit"); - idGroup.consumeAllCaptures( expression -> filters.add( @@ -166,10 +159,8 @@ private List findDynamic(CommandContext commandContext, CaptureGro final CaptureGroup> dynamicGroups = (CaptureGroup>) captures.getGroupIfPresent(DYNAMIC_GROUP_IN); if (dynamicGroups != null) { - Log.info("hit"); dynamicGroups.consumeAllCaptures( expression -> { - Log.info("captured " + expression); final DocValueHasher docValueHasher = new DocValueHasher(); filters.add( new DBFilterBase.INFilter( diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java index 2f8374abcc..bffbababeb 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/AtomicValue.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.service.shredding.model; -import io.quarkus.logging.Log; import java.math.BigDecimal; import java.util.Date; @@ -19,7 +18,6 @@ public record AtomicValue(DocValueType type, String typedFullValue, DocValueHash public static AtomicValue forString(String str) { // For Strings no changes needed, use default prefix+full-value - Log.info("used here"); return create(DocValueType.STRING, str); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java index 544c3c9a27..7dcb34d87b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/shredding/model/DocValueHasher.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.quarkus.logging.Log; import io.stargate.sgv2.jsonapi.exception.ErrorCode; import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.util.JsonUtil; @@ -37,7 +36,6 @@ public class DocValueHasher { final AtomicValues atomics = new AtomicValues(); public DocValueHash hash(JsonNode value) { - Log.info("inside !!"); return switch (value.getNodeType()) { case ARRAY -> arrayHash((ArrayNode) value); case BOOLEAN -> booleanValue(value.booleanValue()).hash(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java index 620b63cc83..40692fa770 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java @@ -726,11 +726,10 @@ public void NonIdIn() throws Exception { }); final FindOperation operation1 = (FindOperation) operation; - System.out.println("geiwo " + operation1.filters()); } @Test - public void NonIdIn_IdEq() throws Exception { + public void NonIdInIdEq() throws Exception { String json = """ { @@ -770,7 +769,7 @@ public void NonIdIn_IdEq() throws Exception { } @Test - public void NonIdIn_IdIn() throws Exception { + public void NonIdInIdIn() throws Exception { String json = """ { @@ -811,7 +810,7 @@ public void NonIdIn_IdIn() throws Exception { } @Test - public void NonIdIn_VSearch() throws Exception { + public void NonIdInVSearch() throws Exception { String json = """ { @@ -851,7 +850,7 @@ public void NonIdIn_VSearch() throws Exception { } @Test - public void NonIdIn_IdIn_VSearch() throws Exception { + public void NonIdInIdInVSearch() throws Exception { String json = """ { @@ -896,7 +895,7 @@ public void NonIdIn_IdIn_VSearch() throws Exception { } @Test - public void descending_sort_with_NonIdIn() throws Exception { + public void descendingSortNonIdIn() throws Exception { String json = """ { @@ -939,7 +938,7 @@ public void descending_sort_with_NonIdIn() throws Exception { } @Test - public void ascending_sort_with_NonIdIn_IdIn() throws Exception { + public void ascendingSortNonIdInIdIn() throws Exception { String json = """ { From de6b5eb337a02d7bd761225f3e147bee6f94f288 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 21 Sep 2023 14:07:46 -0700 Subject: [PATCH 07/17] change back --- docker-compose/docker-compose-dev-mode.yml | 47 +++++++++++----------- nosqlbench/hdrdata.log | 41 ------------------- nosqlbench/hdrstats.log | 5 --- 3 files changed, 23 insertions(+), 70 deletions(-) delete mode 100644 nosqlbench/hdrdata.log delete mode 100644 nosqlbench/hdrstats.log diff --git a/docker-compose/docker-compose-dev-mode.yml b/docker-compose/docker-compose-dev-mode.yml index 25fedd8617..7f19e92d5c 100644 --- a/docker-compose/docker-compose-dev-mode.yml +++ b/docker-compose/docker-compose-dev-mode.yml @@ -8,7 +8,6 @@ services: ports: - "9042:9042" - "8081:8081" - - "8091:8091" mem_limit: 2G environment: - JAVA_OPTS="-Xmx1536M" @@ -23,29 +22,29 @@ services: timeout: 10s retries: 10 -# jsonapi: -# image: ${JSONIMAGE}:${JSONTAG} -# depends_on: -# coordinator: -# condition: service_healthy -# networks: -# - stargate -# ports: -# - "8181:8181" -# mem_limit: 2G -# environment: -# - JAVA_MAX_MEM_RATIO=75 -# - JAVA_INITIAL_MEM_RATIO=50 -# - GC_CONTAINER_OPTIONS=-XX:+UseG1GC -# - QUARKUS_GRPC_CLIENTS_BRIDGE_HOST=coordinator -# - QUARKUS_GRPC_CLIENTS_BRIDGE_PORT=8091 -# - QUARKUS_HTTP_ACCESS_LOG_ENABLED=${REQUESTLOG} -# - QUARKUS_LOG_LEVEL=${LOGLEVEL} -# healthcheck: -# test: curl -f http://localhost:8181/stargate/health || exit 1 -# interval: 5s -# timeout: 10s -# retries: 10 + jsonapi: + image: ${JSONIMAGE}:${JSONTAG} + depends_on: + coordinator: + condition: service_healthy + networks: + - stargate + ports: + - "8181:8181" + mem_limit: 2G + environment: + - JAVA_MAX_MEM_RATIO=75 + - JAVA_INITIAL_MEM_RATIO=50 + - GC_CONTAINER_OPTIONS=-XX:+UseG1GC + - QUARKUS_GRPC_CLIENTS_BRIDGE_HOST=coordinator + - QUARKUS_GRPC_CLIENTS_BRIDGE_PORT=8091 + - QUARKUS_HTTP_ACCESS_LOG_ENABLED=${REQUESTLOG} + - QUARKUS_LOG_LEVEL=${LOGLEVEL} + healthcheck: + test: curl -f http://localhost:8181/stargate/health || exit 1 + interval: 5s + timeout: 10s + retries: 10 networks: stargate: diff --git a/nosqlbench/hdrdata.log b/nosqlbench/hdrdata.log deleted file mode 100644 index 392cabba73..0000000000 --- a/nosqlbench/hdrdata.log +++ /dev/null @@ -1,41 +0,0 @@ -#logging histograms for session nFEb91a -#[Histogram log format version 1.3] -#[StartTime: 1695055647.732 (seconds since epoch), Mon Sep 18 09:47:27 PDT 2023] -"StartTimestamp","Interval_Length","Interval_Max","Interval_Compressed_Histogram" -Tag=bind,1.012,28.996,3.940,HISTFAAAACd42pNpmSzMwMDAwQABLFCaEUza//9v/wEi8HcPJ9P/xRxMAIofCDA= -Tag=execute,1.016,29.003,5262.541,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4jA3zXKTJd7mJkAq40Jsg== -Tag=tries,1.019,29.015,0.000,HISTFAAAAB142pNpmSzMwMDAxAABLFCaEUoz2X+AyQAAP4gCgw== -Tag=result,1.021,29.014,5272.764,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4jA/zXKTPN7mZkAqtAJgQ== -Tag=result_success,1.023,29.027,5272.764,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4jA/zXKTPN7mZkAqtAJgQ== -Tag=read_input,1.030,29.036,0.620,HISTFAAAACd42pNpmSzMwMDAwQABLFCaEUzy//9v/wEi8Pc2G9PdS6xMAIdVCCY= -Tag=strides_servicetime,1.033,29.039,7113.540,HISTFAAAACF42pNpmSzMwMDAwgABMJoRhP8Dgf0HiMDf5epMAITCCEk= -Tag=cycles_servicetime,1.036,29.052,5305.008,HISTFAAAACZ42pNpmSzMwMDAwQABLFCaEYT/A4H9B4hA+1plpt5+ZiYAptUI+g== -Tag=statuscode,1.048,29.055,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAY/CwBKGQOL -Tag=bind,8.309,21.795,6.689,HISTFAAAA/J42j1VTWgdVRTOfO/kvhdr2g6juIhZTmfhqhZa8fLQTRBxMcxSXMrQpbh8vIUEDaGU8ighhBDKo4QsJAQpIiIupDwkSCkiUiSISBFXpahICeIift+588zk3bn3nPN95+eee+flT7aKhYX+1YX0Z9078/Gjs7M3/kyCp/d7eLRs3wR8m+Pz5+z3Pm4vo+LzZBE1vs/wdR/7Pm6cwwPgIKN4jJEmlO8Z7vSwHSh8Bhz3cBf4w1DiNHMW6knUUJDsTzXatEfRw0xQjn/1iDUqiU0j5Y5vSGsjzTWj4m+gMULGVN7puVTvEWcVRmGMKaSzRpaVyCtRPtCiVkCWSCnTT3E17oeLSiE0NCAHalKM3E3l7DV/TShVkFEibvAd7WgrTvdSyc/US/RwTkcHlHs5DjKfdXJ3KW+l+0gklZOnUB3qrJWVKaDScSp/EypjSFZ32QpA/lGYo8gTKnrt8lW1rbJrVtA8HxRomQtyonI+hd5UcU5Q454KkzY6DXeMSzdQdKWZ7ITLaSVLqZo5B2jBBUOTfTKqrO2UhTUDvYdoHdy6VaS2RPIQHSNk2yFGnKdQpI1CM55kk5z7PGiMgesQFU5C8WlD5eyVM9JO/jzBxlKqBUvZut/GY4umBBV1blqJmRlYySoWHvkQ7/suReEHRddJMSiT2O2mF8Ujqn0j8iRPBbNUmVSptAF1YrPUptHxUXF5rF4v5W+p5K3XYth1UC4p8xY2qk9LqnKFQ7qopuvqr4gwDl0UHs3YR+VU07y21I3OCAemYqpE7LI5y7xr2xC7HEpcp9HQIxe+Yj+OzCGtl32SUae2/R9eht+knuhMVMYjNXR+rjXxcyZe4kXcdj3rHauejjZOTazO9udWQqYDxNYx324iRSak6qITkZpKVtftJvCVLjavaZ46kPWW4STztpHoV3AlW8+o6ZpL5R+qTK3JzRdIdxt3xpSpArnX04Y8lpDQiY4nr4qoS+qZLqxIP/9kgv6su5AsugMOHFIZ78exR04ar5Nja6wHj39/4MrpIj4grM9HDbCqGN7DjR7W+GJyM3nfgZx8SJmnewt4qkZZwiPoOiP7fZDgxGQ2yzzHNVzGWzhSWY90885UkHdFtQ67jEPQ10/S7mbY9jquiog33syUw8zv+YiTTNnpW2K4bTKN8v7lov6J2h3IbvM89hcxOYdNgk3sdLQVcLfEL8s4W8Jm3w4vYO9FzFaw9xJ2Ax4XOORnKMePFU5ewL3n8Q42gj5Km6/j41ewm2PrKj67gn8DNi7h0yWsX8PxeUxXsHMRZ4adV/GEcc3exFEPx8uYXMIPr+F4FW/j9CJurmC7j801bBS4ghtLrPV/djUh9g== -Tag=execute,8.310,21.802,5636.096,HISTFAAABe942i2WD2RWexjH39/3d87d3t5eM7P7mt15zcx0J5lkZmaSa5JJJsnMNcnMJJNcc+Wa7mS6MzNJkkwyycxMMpMkSSaTyZXkSiZJupMkSffzPd39e8/5nd/veZ7vn+c5++nP2ZpcLt+W+/6V/P8Z/PuNr+5/vy88/6ter6MuFXU/aCpqRlqUVqIeVmo66KNU1kLQ7DY9yOtRorsFrTToJlvyOhv0d9DnoBvSlrQeNBH0NPrQlVTTlarVRtAprUcVtRz0JdWmNBv0JNV4lW5u06AWErXredDVqElpPK970p0fWNtiTZpP9bsqtK5kKUt2NtFlEXcl1YdEjbqb6LR2uZ530vuiHgW9jGpTr55zL53dprlKjQdidGouusZriZapMdGtoJN6lHfAPXoTNKBzQWc0H9RtFFqQbgfVGOCgHgQdUosTnZOO6BexdjtytRLUqj5Fh9uMehd1NShVzk+vSDtU0l49T3RcF0kprUWzdD3A63qqMT0MehwpoKxhnY16E7XqZxMCQ47v6ai5RI+DZRk2hK+BWNy//wFWbzlDrat7G9WlZwXqzVuA1UqNwGfUqNk9zke9/gkqGNU+l7hsmvp1SaCaCzoIlFO6ETSfqEnXxMJOPUxA8kEC4YuAJjtYrSLDzaDz2U+1jnHqpFYjIpR0Ql8caUSfpAPcsWMfELagJahBvxraDsL3wl+Bo82I1w7bzZoMuhxsqKhxcaSHaI0cPmYIF4O+StOJNgMoYK9HF6QlK4KZdsv0ZaoNgb0DYUb0OuhCsClrWX6TGbPLXisSFXAdVF7w4oUEDo5oKpCJNAd0Leionph8rvao3b7rxj0UiLdI9yEQpGQGZ5y705iatJ/zbWAZTU7IVt9LLbiuVi9NwZpsrTqed5AfDSuIMMyBIaMFIR1WcsJeF1SGyyF2D+hjYHsfC9MONS/bvUiuAZutk7pLLNe5c7u9JeVYhe7IMr6KWgz270UbG3Y2HaLH4DMXN5G95MXJ6CbdiIQ4CPHINGGpYW+XVdulw5ZlwVbt0UyqS8HAyLYYaZYq5zoE3IItfRiJlwy+bE93WdJbFn4+JGMUHrHUCFVWcdXompqo5RQll9GtwMMr1veGXboW9FAOOeFmGIOYBoDCwKpINy1bdhQCImw04alZo/P6ECFpqSkbjWGRN/FTIgu+vpwiO52z7jRbmZprIeFpjYDSx8PdSN6q3+A2byW2LFojrLRSXDMITrCeZ++lkFQAo9Uzq85kFKiyE+xJF7y+yPgxpAKLFhef75ApaGN7n0GV3IURShc9P44Qf8ye2nKoyax++r0NsnoBlrdEhwjS5fZO6J4W08TijHsTjnMwn5lkVH+Ytn0QPWyvVhNk3Id3w2ALvdFIolE9s39GSL8uolWzst/KwcgRmmA5C1xt7yBJR7KTtWxyDNgL+B5Sr9kqX6Kz1rjyCSXcuRlHPLwGWTUpzfbYnEXsQ5slrJ8w1kt0XR0lt0LiF3dcs3V5Jk8lLj7LQ/dTQrBJt1sdHnlfQWk05qq7lFbs1wr9wHCQXmd22EgNvD8b/t+C53InMh/3y+k4qZ9lk7PFgtywrPsxPGyf5nIvKfZBUU7vjR+9Xtu1RejssHnOhwxOq9PWm5mjKBLdXI1uh0FQV7FzzgPjDPxvhAzYqtuyqHMeT8ueFe0slx0bfHnyQVMR91L1CH2WJ11/kvPIamWhqHuO14DmewlWy358W8XNcPa3Kuubbj57eLgzqWaDe6gnm55lyE05XHLNNSTJUWaev2006R4ONeq6sw8TvpdbBDhFmAqW+tmVcpVmj3N2cz83uznepvqkHyeXlUsOYIbIyGjhWARaA6uNtosbv8C563blgv0ZuT9qOj5HltZsSsZlP5jve+7aq/Ugue53R5lQ835dL2Wji2NpRvYYOnHygShlwIPlvCUyxLf8x5J6XlVbkcWQvV+6aOKjXDExXvhfkPbv75anQv9xD6oqv/l4IRKet9yrvF9hVyoobz3RZKU31ml5u8ZTzW33QHn3s6Z+1PxB/Qf7kp7D -Tag=tries,8.314,21.814,0.000,HISTFAAAAB942pNpmSzMwMDAzAABLFCaEUoz2X+Asi7wAwBD1gNf -Tag=result,8.315,21.813,5636.096,HISTFAAABfR42i2WD0TdaxjHz/v9/c5OZy1J0k03x5FJN5NjdzJJMkkmM5mZJMlkZiYzV+a60p0r3WRmkiuTJJlJMkmumZlJJjNzZWauSa6Z7szMZHY/31+3v+e8v/d9nuf753ne8/2vt8tTqeyPqYOv+P//wb/f+Gr992Dh7e/V+jvSdImeBo1HmpSWpD8jPS3SZNAHqVb3gm4d1kZWT2I9LtZajRaktaxGgv4K+hJ0T974LOgmK5E+SnfTmihShV4EXdPzSMVaCfqa1o50K+hFWiOlWjysfi3FatKroJlIY9JoVo+l9UM6oY+sSQtp3VBWW4o5T7DR2KsVWk3rU6ycHsa6rkYXsMdPiTaC3kQqqEuvkppGDmu+SL8orWbNRdoKmo31gBL5GzRkUNOigt2gPv0W9LPmg1r1MlJK96XVoDID7Dc7Z1TnRJR5Tu0873egC97ToG5FDrgT6X2ku4F8Ka1GrrWBatv0Ktag7sRalTYi0wSnNdpMa9iRn0UAzumSRiO9i7QWeHNTJCTKZKS5WJvBslzSbKSvQRcN5P0hleiBVE+CJ8HnWrRdrHK4/gSHRezWSARG2B3UVVXrTeAZqE65wpVIx9WjKYFq1uDOw+RC0EKM4nNO3qinMUBgERCvAyzVs1pGhkWskvyUaQB9r2o9Ynelrmg/QM6gCzjNO3acAsIHWDHcPq0HYtSiTjuVDOgoKZug+6jGgv5A3QCNo+TrIFieswNmZCpoX5qMteMg7Olw1UtBZ3U7ACIysFayFjvGZWt5lSTtFqsc7ncTX7bofgxheWNrVimbWUSRHiSkzgGnOa25wNvnokJenVBTvOzQGe9Apzs0RCBIlWaCfXycSECqVSfnC0AZiq/ITm+jGrqjQn+bgU0ZXyXPT5IfCTNEGOQAmxKN7vspCbtcUA4qB9nQ5xbI46y8JhxqkbY0gk712mrNCFbJd5UeOR9bMvgio4eyim8jE7SWlNzrhR2H6NC8IPuem7rA2Z2k5Z/TWREozyBVCpvctNSca7RqBfQEyZKd2q7bad0nckSfrJv/NoM9R6mfgvZsQAR4YJUWQjxMrWlNuxfKiJunDrOeAWAeFwwlr7JsOuZTSdo8z5ss6lTSIPOJcDkmwGlCVrv4dZGVGTXr5j2P9J1AqXe15eSYEMzVWakJG41hkXVTs3zNvp5JUwFTasvpEJJEmyGesEsA1E2Zx6GwFiKaOTgVa9xsNUDEnNmbVpwhgIsuBwynq7BINTU38Jvj0HKIW2CMXmF0ZlksVkuMuNi8XuajwJFug6p0E0YIsuTpcR4Oht2UH9yhY0n5tHsB6rpInPXk6wZDC4mzMTrUuQVZvOXWhPAUEky6xiGLNWfTXab/P7pDC146C7Qcx/rg+AbtsW0DXSb/lghXzkon1ZvHc5C9kkQuswwY/WR8jLVkcvS6/TA+pNJGNdqPnLbccowq3vcYSxGW2dWfDKIc/GGyWavYjTbL9r6nYiV9Z4s3QOi+e67OI25bHkvPkn7/HPQ5RuCxg4bLaC8DT7Tmuvt0w/Oa4fcw8izYTQzBrYJjepLp/y0xfzPqXHTjXCT3djI661g0eVUAbjXf13nZRopTcJTSXiAJiu3axCUQetLuGQ8JnganrTY1F9Ak8uTP2+D9kFvKzlk3zg1umxchAbbu2VGiMU/aFU+LJpZz+gdkIkC3R10Jfpv31GpkpYLqU77eGgjzyAOoBs3bCFVBdWdZLCVtM8qWJqvuoFb+d1LcsbiMHSUc7SBGB0/6sE8NTG9Yodrkwsry1x19gkN591MJsVrgq5QHtTisIrGtq0gTPZ08T9nQPbwt8LCg6rgHN+eUik/jhyi5FivY6mLd2jMe4i1ESRlYco02yJ8ALpiQL75ZN+1LRmYPqB979tqu1UCZ90WUg6cFj7PlZBBwLJ3QPYxSnHwikvV6ZoxbJGN8Jw+oO46KJsuWpYsSfuJsrT+rvPankKaD6+Wlx/+IfV7qq5+rg/BcdG+zvsVmMpS3FWusyBurtHJEI2nNHvFMef+DFr/TeJf+A099nbw= -Tag=result_success,8.315,21.830,5636.096,HISTFAAABfR42i2WD0TdaxjHz/v9/c5OZy1J0k03x5FJN5NjdzJJMkkmM5mZJMlkZiYzV+a60p0r3WRmkiuTJJlJMkmumZlJJjNzZWauSa6Z7szMZHY/31+3v+e8v/d9nuf753ne8/2vt8tTqeyPqYOv+P//wb/f+Gr992Dh7e/V+jvSdImeBo1HmpSWpD8jPS3SZNAHqVb3gm4d1kZWT2I9LtZajRaktaxGgv4K+hJ0T974LOgmK5E+SnfTmihShV4EXdPzSMVaCfqa1o50K+hFWiOlWjysfi3FatKroJlIY9JoVo+l9UM6oY+sSQtp3VBWW4o5T7DR2KsVWk3rU6ycHsa6rkYXsMdPiTaC3kQqqEuvkppGDmu+SL8orWbNRdoKmo31gBL5GzRkUNOigt2gPv0W9LPmg1r1MlJK96XVoDID7Dc7Z1TnRJR5Tu0873egC97ToG5FDrgT6X2ku4F8Ka1GrrWBatv0Ktag7sRalTYi0wSnNdpMa9iRn0UAzumSRiO9i7QWeHNTJCTKZKS5WJvBslzSbKSvQRcN5P0hleiBVE+CJ8HnWrRdrHK4/gSHRezWSARG2B3UVVXrTeAZqE65wpVIx9WjKYFq1uDOw+RC0EKM4nNO3qinMUBgERCvAyzVs1pGhkWskvyUaQB9r2o9Ynelrmg/QM6gCzjNO3acAsIHWDHcPq0HYtSiTjuVDOgoKZug+6jGgv5A3QCNo+TrIFieswNmZCpoX5qMteMg7Olw1UtBZ3U7ACIysFayFjvGZWt5lSTtFqsc7ncTX7bofgxheWNrVimbWUSRHiSkzgGnOa25wNvnokJenVBTvOzQGe9Apzs0RCBIlWaCfXycSECqVSfnC0AZiq/ITm+jGrqjQn+bgU0ZXyXPT5IfCTNEGOQAmxKN7vspCbtcUA4qB9nQ5xbI46y8JhxqkbY0gk712mrNCFbJd5UeOR9bMvgio4eyim8jE7SWlNzrhR2H6NC8IPuem7rA2Z2k5Z/TWREozyBVCpvctNSca7RqBfQEyZKd2q7bad0nckSfrJv/NoM9R6mfgvZsQAR4YJUWQjxMrWlNuxfKiJunDrOeAWAeFwwlr7JsOuZTSdo8z5ss6lTSIPOJcDkmwGlCVrv4dZGVGTXr5j2P9J1AqXe15eSYEMzVWakJG41hkXVTs3zNvp5JUwFTasvpEJJEmyGesEsA1E2Zx6GwFiKaOTgVa9xsNUDEnNmbVpwhgIsuBwynq7BINTU38Jvj0HKIW2CMXmF0ZlksVkuMuNi8XuajwJFug6p0E0YIsuTpcR4Oht2UH9yhY0n5tHsB6rpInPXk6wZDC4mzMTrUuQVZvOXWhPAUEky6xiGLNWfTXab/P7pDC146C7Qcx/rg+AbtsW0DXSb/lghXzkon1ZvHc5C9kkQuswwY/WR8jLVkcvS6/TA+pNJGNdqPnLbccowq3vcYSxGW2dWfDKIc/GGyWavYjTbL9r6nYiV9Z4s3QOi+e67OI25bHkvPkn7/HPQ5RuCxg4bLaC8DT7Tmuvt0w/Oa4fcw8izYTQzBrYJjepLp/y0xfzPqXHTjXCT3djI661g0eVUAbjXf13nZRopTcJTSXiAJiu3axCUQetLuGQ8JnganrTY1F9Ak8uTP2+D9kFvKzlk3zg1umxchAbbu2VGiMU/aFU+LJpZz+gdkIkC3R10Jfpv31GpkpYLqU77eGgjzyAOoBs3bCFVBdWdZLCVtM8qWJqvuoFb+d1LcsbiMHSUc7SBGB0/6sE8NTG9Yodrkwsry1x19gkN591MJsVrgq5QHtTisIrGtq0gTPZ08T9nQPbwt8LCg6rgHN+eUik/jhyi5FivY6mLd2jMe4i1ESRlYco02yJ8ALpiQL75ZN+1LRmYPqB979tqu1UCZ90WUg6cFj7PlZBBwLJ3QPYxSnHwikvV6ZoxbJGN8Jw+oO46KJsuWpYsSfuJsrT+rvPankKaD6+Wlx/+IfV7qq5+rg/BcdG+zvsVmMpS3FWusyBurtHJEI2nNHvFMef+DFr/TeJf+A099nbw= -Tag=read_input,8.328,21.833,0.065,HISTFAAAAQx42j1QsUpDQRB8N7fvPF+iBGxtrPwGq3zCsvhLSrC2EIuQ4pHS2kKsrMTawkIsxCqNQeRhFWfvQu4tzN7O7Lzhji+uj5omzJp6ZIuhwmYz/a7dQ8Y8YgjZkkK0U4GJTrwfQw9Vsuas5JLmRExQsTFpg0+zdlBY5i0TO3EQTZb7IG6Jntb0MjFHXyMv1KY+oCBFjLAMZHwJX3tS9rycUefIKBZ+MX5sngH3wiMT4QVVU1viIrq9yToKcdgWzUkXP0YsXsvgglWLJ/HxPJaEJtXG/P9/ZTwUfwp3ZT64aUsgxTriYx+3Ld5G3tf4rNcD3Gfc8TFxjp+JL/6eyecUl6dYnWA2wvsV8A+IWlao -Tag=strides_servicetime,8.329,21.835,10710.680,HISTFAAAAZh42i2RwStEURTG5/zuu+81mIaEhCRkIWsLyUKSLCQLSbKSLP0BktCkSa9pkqxkIVlYSJIsNM3CQtM0SZpkYWGhaUKykMS5ceveW1/nnu93vtu8lq6NRIhG/pb3f4tu86Nr4PVPyIcd7HTxHePMJxFjr5oyZKsYIOuzIpzHnVDLewU5KAmWZMBVBadCJ98BJcuLoVjPLA1EufZ4gFXIBKQMO54rf4ZlHoRJ6tgUMtBNCc4M98KmZU8IhUNRn1XhAippJSPOsYu08CmMqvLhcRowrW1y+szTqtBnF/qIkbZcCuPUUBbeIK+nMONcj2GOD2EDWvg09KtND2O0sUTBY5gJApLCjejrNtr5QQcJPabo5d2yDYdwIqxbxejmUbgTCIErwajpELeQRBHnGOTcUAiYdyBNHAmXuInnFXKGIuzj8FVPCFuw4BA16zgHaKJRDiwFnw2rSesQT+rECI0OoYdrSMCXS1KB4/pLi2qqNKFhHVJQjCpPzlezbVe1G5DXPsY5pSy3VZSNK054TsmKpqmfnAr4BT9Bc+M= -Tag=cycles_servicetime,8.330,21.850,5636.358,HISTFAAABel42i2WAWSVexjGz//5vq+zrzqOyXGkm5nJ7M7MZJKZzByTmUwymckkyXVdVzJJrtlNMnNkMkkyRzJJkkkmM8lkkkmSJDPJlXRz5Zpcu7/nq/M5Z9/3//7v+z7v8z7v+99Pf07vyOXSA7nvn/jH3+DvJp8Df39f+DK1S1cjTRZ0N6ga6Yq0JK1EelSnhaBV6YD+0Gaqd6nexKpt09pu/RdUSzUT9CDoa9C89E16FHQx6HWkD9Jcost12qmVoBN6H2mb7gRtJlqTbgYtJ5or6tJWHdaDWAf1JGg60mVpPNUTaXGLWv10gc0JPuM3Ul5Xgt4FbUSqSWf0KlY1MWQMx4N26640EelrQZ+Cbkdq1pCe4TBSdatT6VejOjQbGdF0rPlIE4mWgob1KlVVvFsPOqbJQLI13z2MlNMtaTko1UensRqgosWB4KhfPdLloAsRLvDToKPSfNDnSGuRbgTZ/FGkqyKXsipaj0F0NdY9aTXSpWByt+lJonOGtBSpC4QnncPHSA+d0aR0SHJNarFhYPmLagAXIaHi7RYVdF9qU72eBpt16/U2kLRpRXpYp9OA+CT9rrdBp3gq2SqvRWmvc1w0SyMmutWAhzWgMd0KuhXjo+bg7VqJldgH/LwOmNkgJd79YNbJv17HQTVmQlvBfMZyGOHakHr1q6pBfdzMRfocyG/IVO2BxIMsFmC5GVj7oXWPGbmROW3Rq0DQfu3AoIIrKEFpVHIq1oegXboQNKhLcvUHjaETSG8caERFXc8UF3QtWIe74ISHCy7djN/MiT2NumFZF7gdz+QwCrJJv54OsDDrFJ6LOxB1kbXuBfwU7IY64Ocf167s14TvYM+iszuEWaf2wQeZ26aXgmJTNIRePZVjlNiwj82zbooSdelw1bMGIp+y22PAkJq5TtFBJ/Tlu7p2GlzRklyMwdIH6AzWEeqdg3rar8d856lIqoXM3dvIe2hSmmfUrgGyA/KpbrPuCL+dhFwLmor0Iuh5hItByE8JNBHAiV3F9WgnDAt3LNdezSRusfsR5SbqYYzm/eeYi08DHNDVYIU0OuJZ0tzhxNHZbcs65XoW9DL492m2eB4gK363lHX8dVsNsK2iJuQwbdJymjKrTRbQkgGCxlnvRuPjQkO0TDNqbGS1DTmlLk4ODwn1OwJfA2xt5XVkDuq5BRtds1P/BkdYDjH9VorPs7k+q0kLJl2Yz8TWWhMW7e6akls6Dz9tbtCSGXfPdZtoYHS4x6ohy7KAPXMvHsPXOd5FRBv0jCzD2LAHTkK0CmsFrlZ8tqLDNpKHAapecFNMBU+cDowPES3vMh2mabohNaUv+0A2l7EJvy2uQ4QKq44xRtBayNrwpN05LVg7BCFNXKN8xxgOryyiUzhC8RXLtd9VL/tnCKngkfFYb9XD6P64nbWC22j0xwTAomYlMZmrVhe7GNz6Frlxzuqbh2kp46jJ7TvrOh+1eCOl8Q1TOAwvx7L82XyUMqIFRj4bUAh9scEhE5PXpCkpw8GXPNuAu2Cwy5Y2JxDjbEZ6L0dYTTzIRkh4QJvB2thP1OO6LfrphM+oTc8hyjPngdKHZGnp0+DoAUUFq5wH1pBPrb+8owDMbhf8kqdEGdsVcwTnIxQxcYs1WPXHCVkk2E0Py3Nm6IUnWtFQe/By0azcD9mYaMHkQ5ZgigQmPJiabHgSH3mcDwMCPq2HJffibhD0ZkNj0GdREYencNlDcfKZAutJpASPIOziNsfrY9DWgK8hfA7C7CjPPpnq8ZNg1sra3u+BxdIveBngkfxOY5Xg3iiS7Cpj7aNkJOupFI87fRL1KxcnABvOJNrDy06gNuCl3kPY4yDP702frHdNRkKgo67JhpGsWDMIsYK/xy5lBRSuAwX8jZ3NLlHe3TVjcTxOsMlBbfa/xWO34ojDTIZsFDToo/R4S5YhNbnnITTgu0ZQNXl4vrPbLleGkSfyzU6TorXDHFxItBxrnfkZ6ZphP4s1WaeXIvv57RpPNLvd/658/lnrDVqt6H8PWaFD -Tag=statuscode,8.331,21.865,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AQBKhQP3 -Tag=statuscode,8.331,21.865,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AQBKhQP3 -Tag=statuscode,8.337,21.859,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HABKcwPl -Tag=statuscode,8.338,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/FABKbwPh -Tag=statuscode,8.338,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/GABKawPd -Tag=statuscode,8.338,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/DABKcQPj -Tag=statuscode,8.338,21.859,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HgBKgwP1 -Tag=statuscode,8.338,21.859,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/FABKbwPh -Tag=statuscode,8.339,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HgBKgwP1 -Tag=statuscode,8.339,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AQBKhQP3 -Tag=statuscode,8.339,21.858,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AABKZQPX -Tag=statuscode,8.340,21.857,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/BgBKfQPv -Tag=statuscode,8.341,21.856,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/CgBKeQPr -Tag=statuscode,8.341,21.856,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/HgBKgwP1 -Tag=statuscode,8.342,21.855,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/BQBKjQP/ -Tag=statuscode,8.343,21.854,0.000,HISTFAAAAB142pNpmSzMwMDAzIAKGKH0f/sPEAZ/AyMATsEECQ== -Tag=statuscode,8.345,21.852,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/CgBKeQPr -Tag=statuscode,8.348,21.850,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AgBKdQPn -Tag=statuscode,8.349,21.849,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAa/FwBKXwPR -Tag=statuscode,8.354,21.844,0.000,HISTFAAAABx42pNpmSzMwMDAxIAKGKH0f/sPEAZ/AABKZQPX diff --git a/nosqlbench/hdrstats.log b/nosqlbench/hdrstats.log deleted file mode 100644 index 2a46985098..0000000000 --- a/nosqlbench/hdrstats.log +++ /dev/null @@ -1,5 +0,0 @@ -#logging stats for session nFEa4dy -#[Histogram log format version 1.0] -#[StartTime: 1695055365.611 (seconds since epoch), Mon Sep 18 09:42:45 PDT 2023] -#[TimeUnit: NANOSECONDS] -#Tag,Interval_Start,Interval_Length,count,min,p25,p50,p75,p90,p95,p98,p99,p999,p9999,max From 31d0d5908eeb39cafc2131bfe964e7230791d153 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 21 Sep 2023 14:10:27 -0700 Subject: [PATCH 08/17] change back --- nosqlbench/http-jsonapi-vector-crud.yaml | 35 ++++++++++++------------ pom.xml | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/nosqlbench/http-jsonapi-vector-crud.yaml b/nosqlbench/http-jsonapi-vector-crud.yaml index b94a61e106..54767930f8 100644 --- a/nosqlbench/http-jsonapi-vector-crud.yaml +++ b/nosqlbench/http-jsonapi-vector-crud.yaml @@ -3,7 +3,6 @@ min_version: "5.17.3" # Example command line # Against AstraDB # nb5 -v http-jsonapi-vector-crud docscount=1000 threads=20 jsonapi_host=Your-AstraDB-Host auth_token=Your-AstraDB-Token jsonapi_port=443 protocol=https path_prefix=/api/json namespace=Your-Keyspace -# java -jar ~/nb5.jar -v http-jsonapi-vector-crud docscount=10000 threads=100 jsonapi_host=cc6478dd-2d0e-4576-87f1-fa3cbd3e10bb auth_token=AstraCS:OssyPlglELzWhYaIJgdqluCN:c136258f28ab5348f6288d06b0e794c72b8c8d89e163a62b0ab5a3bff61a522b-us-east-2.apps.astra-dev.datastax.com jsonapi_port=443 protocol=https path_prefix=/api/json namespace=photography # Against local JSON API # nb5 -v http-jsonapi-vector-crud jsonapi_host=localhost docscount=1000 threads=20 @@ -17,9 +16,9 @@ scenarios: default: schema: run driver=http tags==block:schema threads==1 cycles==UNDEF write: run driver=http tags==name:"write.*" cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn -# read: run driver=http tags==name:"read.*" cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn -# update: run driver=http tags==name:"update.*" cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn -# delete: run driver=http tags==name:"delete.*" cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn + read: run driver=http tags==name:"read.*" cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn + update: run driver=http tags==name:"update.*" cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn + delete: run driver=http tags==name:"delete.*" cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,500)) threads=auto errors=timer,warn bindings: # To enable an optional weighted set of hosts in place of a load balancer @@ -45,20 +44,20 @@ bindings: blocks: schema: ops: -# create-namespace: -# method: POST -# uri: <>://{weighted_hosts}:<><>/v1 -# Accept: "application/json" -# X-Cassandra-Request-Id: "{request_id}" -# X-Cassandra-Token: "{token}" -# Content-Type: "application/json" -# ok-body: ".*\"ok\":1.*" -# body: >2 -# { -# "createNamespace": { -# "name": "<>" -# } -# } + create-namespace: + method: POST + uri: <>://{weighted_hosts}:<><>/v1 + Accept: "application/json" + X-Cassandra-Request-Id: "{request_id}" + X-Cassandra-Token: "{token}" + Content-Type: "application/json" + ok-body: ".*\"ok\":1.*" + body: >2 + { + "createNamespace": { + "name": "<>" + } + } delete-collection: method: POST diff --git a/pom.xml b/pom.xml index 431d00c572..165d939bae 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ com.bpodgursky jbool_expressions - 1.23 + 1.24 io.quarkus From f4c672bf81f244c06eb6aea5355f99767926d8f1 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 21 Sep 2023 14:11:34 -0700 Subject: [PATCH 09/17] change back --- .../command/deserializers/FilterClauseDeserializer.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java index c0c6be5907..5f990b96a5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/deserializers/FilterClauseDeserializer.java @@ -89,11 +89,6 @@ private void validate(String path, FilterOperation filterOperation) { if (filterOperation.operator() instanceof ValueComparisonOperator valueComparisonOperator) { switch (valueComparisonOperator) { case IN -> { - // if (!path.equals(DocumentConstants.Fields.DOC_ID)) { - // throw new JsonApiException( - // ErrorCode.INVALID_FILTER_EXPRESSION, "Can use $in operator only on _id - // field"); - // } if (filterOperation.operand().value() instanceof List list) { if (list.size() > operationsConfig.defaultPageSize()) { From 3797115ef8c03337bc94fa0832361c24b5a775be Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Fri, 22 Sep 2023 14:16:55 -0700 Subject: [PATCH 10/17] version bump manually --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 049b4eb866..c206e37bbd 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.stargate sgv2-api-parent - 2.1.0-ALPHA-10 + 2.1.0-ALPHA-11 sgv2-jsonapi 1.0.0-BETA-1-SNAPSHOT From 4e04b6899aa2019626831f55deafad329fc15ab7 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Fri, 22 Sep 2023 15:49:03 -0700 Subject: [PATCH 11/17] remove mock commandContext --- .../service/resolver/model/impl/FindCommandResolverTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java index fcae161422..1777c694ac 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java @@ -724,7 +724,7 @@ public void noFilterConditionWithProjection() throws Exception { @Nested class FindCommandResolveWithINOperator { - @Mock CommandContext commandContext; + CommandContext commandContext = CommandContext.empty(); @Test public void NonIdIn() throws Exception { From 5d085c6f7718ce269f63f838cfa83fab9a30fd5c Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Mon, 25 Sep 2023 11:16:40 -0700 Subject: [PATCH 12/17] rename InFilter --- .../operation/model/impl/DBFilterBase.java | 8 +++--- .../operation/model/impl/FindOperation.java | 6 ++-- .../impl/matcher/FilterableResolver.java | 4 +-- .../model/impl/FindCommandResolverTest.java | 28 +++++++++---------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java index 2d2daf0037..81e1d43f40 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java @@ -290,9 +290,9 @@ boolean canAddField() { * Filters db documents based on non document id field values if id use "$in" operator, use * IDFilter, since partition key 'or' is not supported besides id, "$in" operator, use INFilter */ - public static class INFilter extends DBFilterBase { + public static class InFilter extends DBFilterBase { private final Object arrayValue; - protected final INFilter.Operator operator; + protected final InFilter.Operator operator; @Override JsonNode asJson(JsonNodeFactory nodeFactory) { @@ -308,7 +308,7 @@ public enum Operator { IN; } - public INFilter(INFilter.Operator operator, String path, Object arrayValue) { + public InFilter(InFilter.Operator operator, String path, Object arrayValue) { super(path); this.arrayValue = arrayValue; this.operator = operator; @@ -318,7 +318,7 @@ public INFilter(INFilter.Operator operator, String path, Object arrayValue) { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - INFilter inFilter = (INFilter) o; + InFilter inFilter = (InFilter) o; return operator == inFilter.operator && Objects.equals(arrayValue, inFilter.arrayValue); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index 28d5bc4491..58a9d3b5b8 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -389,7 +389,7 @@ private List buildSelectQueries(DBFilterBase.IDFilter add AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { - if (filter instanceof DBFilterBase.INFilter) { + if (filter instanceof DBFilterBase.InFilter) { hasInFilterBesidesIdField.set(true); return; // need to break } @@ -595,7 +595,7 @@ private List buildSortedSelectQueries( AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { - if (filter instanceof DBFilterBase.INFilter) { + if (filter instanceof DBFilterBase.InFilter) { hasInFilterBesidesIdField.set(true); return; // need to break } @@ -716,7 +716,7 @@ private List> buildConditionExpressions( // if we have id filter overwrite ignore existing IDFilter boolean idFilterOverwrite = additionalIdFilter != null; for (DBFilterBase filter : filters) { - if (filter instanceof DBFilterBase.INFilter inFilter) { + if (filter instanceof DBFilterBase.InFilter inFilter) { List conditions = inFilter.getAll(); if (!conditions.isEmpty()) { List> variableConditions = diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java index c9a758b593..83470a5067 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/matcher/FilterableResolver.java @@ -163,8 +163,8 @@ private List findDynamic(CommandContext commandContext, CaptureGro expression -> { final DocValueHasher docValueHasher = new DocValueHasher(); filters.add( - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, expression.path(), expression.value())); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, expression.path(), expression.value())); }); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java index 1777c694ac..c1d8812556 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindCommandResolverTest.java @@ -745,8 +745,8 @@ public void NonIdIn() throws Exception { FindOperation.class, find -> { DBFilterBase filter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); assertThat(find.objectMapper()).isEqualTo(objectMapper); assertThat(find.commandContext()).isEqualTo(commandContext); assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); @@ -784,8 +784,8 @@ public void NonIdInIdEq() throws Exception { FindOperation.class, find -> { DBFilterBase inFilter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); DBFilterBase idFilter = new DBFilterBase.IDFilter( DBFilterBase.IDFilter.Operator.EQ, DocumentId.fromString("id1")); @@ -824,8 +824,8 @@ public void NonIdInIdIn() throws Exception { FindOperation.class, find -> { DBFilterBase inFilter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); DBFilterBase idFilter = new DBFilterBase.IDFilter( DBFilterBase.IDFilter.Operator.IN, @@ -867,8 +867,8 @@ public void NonIdInVSearch() throws Exception { FindOperation.class, find -> { DBFilterBase inFilter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); float[] vector = new float[] {0.15f, 0.1f, 0.1f}; assertThat(find.objectMapper()).isEqualTo(objectMapper); assertThat(find.commandContext()).isEqualTo(commandContext); @@ -908,8 +908,8 @@ public void NonIdInIdInVSearch() throws Exception { FindOperation.class, find -> { DBFilterBase inFilter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); DBFilterBase idFilter = new DBFilterBase.IDFilter( DBFilterBase.IDFilter.Operator.IN, @@ -955,8 +955,8 @@ public void descendingSortNonIdIn() throws Exception { find -> { FindOperation.OrderBy orderBy = new FindOperation.OrderBy("name", false); DBFilterBase inFilter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); assertThat(find.objectMapper()).isEqualTo(objectMapper); assertThat(find.commandContext()).isEqualTo(commandContext); assertThat(find.projection()).isEqualTo(DocumentProjector.identityProjector()); @@ -999,8 +999,8 @@ public void ascendingSortNonIdInIdIn() throws Exception { find -> { FindOperation.OrderBy orderBy = new FindOperation.OrderBy("name", true); DBFilterBase inFilter = - new DBFilterBase.INFilter( - DBFilterBase.INFilter.Operator.IN, "name", List.of("test1", "test2")); + new DBFilterBase.InFilter( + DBFilterBase.InFilter.Operator.IN, "name", List.of("test1", "test2")); DBFilterBase idFilter = new DBFilterBase.IDFilter( DBFilterBase.IDFilter.Operator.IN, From 7cd2c724e264dce2e5371b82108ec2b159cd3f51 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Mon, 25 Sep 2023 23:05:14 -0700 Subject: [PATCH 13/17] refining --- .../operation/model/impl/DBFilterBase.java | 7 ++++-- .../operation/model/impl/FindOperation.java | 8 ++++-- .../jsonapi/api/v1/FindIntegrationTest.java | 25 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java index 81e1d43f40..68e7b937f9 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java @@ -16,7 +16,10 @@ import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId; import io.stargate.sgv2.jsonapi.util.JsonUtil; import java.math.BigDecimal; -import java.util.*; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -363,7 +366,7 @@ public List getAll() { */ public abstract static class SetFilterBase extends DBFilterBase { public enum Operator { - CONTAINS, + CONTAINS; } protected final String columnName; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index 58a9d3b5b8..c3a693bf5e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -376,8 +376,6 @@ public ReadDocument getNewDocument() { return ReadDocument.from(documentId, null, rootNode); } - // builds select query - /** * Builds select query based on filters and additionalIdFilter overrides. * @@ -386,6 +384,9 @@ public ReadDocument getNewDocument() { * buildConditions method. */ private List buildSelectQueries(DBFilterBase.IDFilter additionalIdFilter) { + // if the query has "$in" operator for non-id field, buildCondition should return List of + // Expression + // that is the reason having this boolean AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { @@ -592,6 +593,9 @@ private QueryOuterClass.Query getVectorSearchQueryByExpression( */ private List buildSortedSelectQueries( DBFilterBase.IDFilter additionalIdFilter) { + // if the query has "$in" operator for non-id field, buildCondition should return List of + // Expression + // that is the reason having this boolean AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java index 30560aa1ef..6e7083a1e1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java @@ -409,6 +409,31 @@ public void inConditionNonIdField() { .body("data.documents[0]", jsonEquals(expected1)); } + @Test + public void inConditionNonIdFieldMulti() { + String json = + """ + { + "find": { + "filter" : { + "username" : {"$in" : ["user1", "user2"]} + } + } + } + """; + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("data.documents", hasSize(2)) + .body("status", is(nullValue())) + .body("errors", is(nullValue())); + } + @Test public void inConditionNonIdFieldIdField() { String json = From 9b77a3693cb36b17e038d3ba03223ca3e9ad9513 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Tue, 26 Sep 2023 18:00:54 -0700 Subject: [PATCH 14/17] resolve pr comments --- .../config/constants/DocumentConstants.java | 6 ++++ .../sgv2/jsonapi/exception/ErrorCode.java | 2 +- .../bridge/executor/NamespaceCache.java | 4 +-- .../operation/model/impl/DBFilterBase.java | 29 ++++++------------- .../operation/model/impl/FindOperation.java | 10 ++++--- .../jsonapi/api/v1/FindIntegrationTest.java | 17 +++++++++-- 6 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java b/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java index 985fa16b94..449b1c71c0 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java @@ -9,6 +9,12 @@ interface Fields { /** Primary key for Documents stored; has special handling for many operations. */ String DOC_ID = "_id"; + /** + * Atomic values are added to the array_contains field to support $eq on both atomic value and + * array element + */ + String DATA_CONTAINS = "array_contains"; + /** Physical table column name that stores the vector field. */ String VECTOR_SEARCH_INDEX_COLUMN_NAME = "query_vector_value"; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/exception/ErrorCode.java b/src/main/java/io/stargate/sgv2/jsonapi/exception/ErrorCode.java index c0c68277c0..6679d7d325 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/exception/ErrorCode.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/exception/ErrorCode.java @@ -95,7 +95,7 @@ public enum ErrorCode { VECTOR_SEARCH_NOT_SUPPORTED("Vector search is not enabled for the collection "), - VECTOR_SEARCH_INVALID_FUCTION_NAME("Invalid vector search function name "), + VECTOR_SEARCH_INVALID_FUNCTION_NAME("Invalid vector search function name: "), VECTOR_SEARCH_SIMILARITY_PROJECTION_NOT_SUPPORTED( "$similarity projection is not supported for this command"), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/bridge/executor/NamespaceCache.java b/src/main/java/io/stargate/sgv2/jsonapi/service/bridge/executor/NamespaceCache.java index b67118c4db..8349c8daf6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/bridge/executor/NamespaceCache.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/bridge/executor/NamespaceCache.java @@ -164,8 +164,8 @@ public static SimilarityFunction fromString(String similarityFunction) { case "euclidean" -> EUCLIDEAN; case "dot_product" -> DOT_PRODUCT; default -> throw new JsonApiException( - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME, - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME.getMessage() + similarityFunction); + ErrorCode.VECTOR_SEARCH_INVALID_FUNCTION_NAME, + ErrorCode.VECTOR_SEARCH_INVALID_FUNCTION_NAME.getMessage() + similarityFunction); }; } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java index 68e7b937f9..8c3a5d6911 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DBFilterBase.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.service.operation.model.impl; +import static io.stargate.sgv2.jsonapi.config.constants.DocumentConstants.Fields.DATA_CONTAINS; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; @@ -78,12 +80,6 @@ public enum Operator { protected final DBFilterBase.MapFilterBase.Operator operator; private final T value; - /** - * Atomic values are added to the array_contains field to support $eq on both atomic value and - * array element - */ - private static final String DATA_CONTAINS = "array_contains"; - protected MapFilterBase( String columnName, String key, MapFilterBase.Operator operator, T value) { super(key); @@ -290,11 +286,10 @@ boolean canAddField() { } /** - * Filters db documents based on non document id field values if id use "$in" operator, use - * IDFilter, since partition key 'or' is not supported besides id, "$in" operator, use INFilter + * based on values of fields other than document id: for filtering on non-id field use InFilter. */ public static class InFilter extends DBFilterBase { - private final Object arrayValue; + private final List arrayValue; protected final InFilter.Operator operator; @Override @@ -306,12 +301,12 @@ JsonNode asJson(JsonNodeFactory nodeFactory) { boolean canAddField() { return false; } - + // IN operator for non-id field filtering public enum Operator { IN; } - public InFilter(InFilter.Operator operator, String path, Object arrayValue) { + public InFilter(InFilter.Operator operator, String path, List arrayValue) { super(path); this.arrayValue = arrayValue; this.operator = operator; @@ -330,16 +325,13 @@ public int hashCode() { return Objects.hash(arrayValue, operator); } - private static final String DATA_CONTAINS = "array_contains"; - @Override public BuiltCondition get() { - // For IN filter we always use getALL() method - return null; + throw new UnsupportedOperationException("For IN filter we always use getALL() method"); } public List getAll() { - List values = (List) arrayValue; + List values = arrayValue; switch (operator) { case IN: if (values.isEmpty()) return List.of(); @@ -360,10 +352,7 @@ public List getAll() { } } - /** - * DB filter / condition for testing a set value Note: we can only do CONTAINS until SAI indexes - * are updated Now, cassandra supports or operation, we can add $in - */ + /** DB filter / condition for testing a set value */ public abstract static class SetFilterBase extends DBFilterBase { public enum Operator { CONTAINS; diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index c3a693bf5e..da07fda7a6 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -504,8 +504,9 @@ private QueryOuterClass.Query getVectorSearchQuery(List conditio } default -> { throw new JsonApiException( - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME, - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME.getMessage()); + ErrorCode.VECTOR_SEARCH_INVALID_FUNCTION_NAME, + ErrorCode.VECTOR_SEARCH_INVALID_FUNCTION_NAME.getMessage() + + commandContext().similarityFunction()); } } } else { @@ -569,8 +570,9 @@ private QueryOuterClass.Query getVectorSearchQueryByExpression( } default -> { throw new JsonApiException( - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME, - ErrorCode.VECTOR_SEARCH_INVALID_FUCTION_NAME.getMessage()); + ErrorCode.VECTOR_SEARCH_INVALID_FUNCTION_NAME, + ErrorCode.VECTOR_SEARCH_INVALID_FUNCTION_NAME.getMessage() + + commandContext().similarityFunction()); } } } else { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java index 6e7083a1e1..e9c5a50cd8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java @@ -87,6 +87,7 @@ public void setUp() { "insertOne": { "document": { "_id": "doc4", + "username" : "user4", "indexedObject" : { "0": "value_0", "1": "value_1" } } } @@ -263,7 +264,7 @@ public void inCondition() { """; String expected2 = """ - {"_id":"doc4", "indexedObject":{"0":"value_0","1":"value_1"}} + {"_id":"doc4", "username":"user4", "indexedObject":{"0":"value_0","1":"value_1"}} """; given() @@ -416,11 +417,19 @@ public void inConditionNonIdFieldMulti() { { "find": { "filter" : { - "username" : {"$in" : ["user1", "user2"]} + "username" : {"$in" : ["user1", "user4"]} } } } """; + String expected1 = + """ + {"_id":"doc1", "username":"user1", "active_user":true, "date" : {"$date": 1672531200000}} + """; + String expected2 = + """ + {"_id":"doc4", "username":"user4", "indexedObject":{"0":"value_0","1":"value_1"}} + """; given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) .contentType(ContentType.JSON) @@ -431,7 +440,8 @@ public void inConditionNonIdFieldMulti() { .statusCode(200) .body("data.documents", hasSize(2)) .body("status", is(nullValue())) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.documents", containsInAnyOrder(jsonEquals(expected1), jsonEquals(expected2))); } @Test @@ -656,6 +666,7 @@ public void withEqSubDocWithIndex() { """ { "_id": "doc4", + "username":"user4", "indexedObject" : { "0": "value_0", "1": "value_1" } } """; From 63f0251be12de3e6cf140c3a9c81007e488c5141 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 27 Sep 2023 09:28:30 -0700 Subject: [PATCH 15/17] leave code refactor for next stargate-release --- .../service/operation/model/impl/FindOperation.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index da07fda7a6..7fbc3a3b69 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -387,6 +387,8 @@ private List buildSelectQueries(DBFilterBase.IDFilter add // if the query has "$in" operator for non-id field, buildCondition should return List of // Expression // that is the reason having this boolean + // TODO queryBuilder change for where(Expression> instead of List> AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { @@ -397,6 +399,7 @@ private List buildSelectQueries(DBFilterBase.IDFilter add }); if (hasInFilterBesidesIdField.get()) { + // This if block handles filter with "$in" for non-id field List> expressions = buildConditionExpressions(additionalIdFilter); if (expressions == null) { return List.of(); @@ -425,8 +428,8 @@ private List buildSelectQueries(DBFilterBase.IDFilter add }); return queries; - } else { + // This if block handles filter with no "$in" for non-id field List> conditions = buildConditions(additionalIdFilter); if (conditions == null) { return List.of(); @@ -598,6 +601,8 @@ private List buildSortedSelectQueries( // if the query has "$in" operator for non-id field, buildCondition should return List of // Expression // that is the reason having this boolean + // TODO queryBuilder change for where(Expression> instead of List> AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { @@ -607,6 +612,7 @@ private List buildSortedSelectQueries( } }); if (hasInFilterBesidesIdField.get()) { + // This if block handles filter with "$in" for non-id field List> expressions = buildConditionExpressions(additionalIdFilter); if (expressions == null) { return List.of(); @@ -633,6 +639,7 @@ private List buildSortedSelectQueries( return queries; } else { + // This if block handles filter with "$in" for non-id field List> conditions = buildConditions(additionalIdFilter); if (conditions == null) { return List.of(); From 3b3bc03f5699ddb395acbff88dc840b1314969eb Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 27 Sep 2023 09:33:08 -0700 Subject: [PATCH 16/17] format --- .../jsonapi/service/operation/model/impl/FindOperation.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index 7fbc3a3b69..f398c43e08 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -388,7 +388,8 @@ private List buildSelectQueries(DBFilterBase.IDFilter add // Expression // that is the reason having this boolean // TODO queryBuilder change for where(Expression> instead of List> + // TODO then we can fully rely on List> instead of + // List> AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { @@ -602,7 +603,8 @@ private List buildSortedSelectQueries( // Expression // that is the reason having this boolean // TODO queryBuilder change for where(Expression> instead of List> + // TODO then we can fully rely on List> instead of + // List> AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); filters.forEach( filter -> { From 74d4d6b707d84a941c9d0d8813c42d7e96cb5d75 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 27 Sep 2023 09:49:08 -0700 Subject: [PATCH 17/17] remove atomicBoolean variable --- .../operation/model/impl/FindOperation.java | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java index f398c43e08..ebfee0a6fb 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java @@ -27,7 +27,7 @@ import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -389,17 +389,10 @@ private List buildSelectQueries(DBFilterBase.IDFilter add // that is the reason having this boolean // TODO queryBuilder change for where(Expression> instead of - // List> - AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); - filters.forEach( - filter -> { - if (filter instanceof DBFilterBase.InFilter) { - hasInFilterBesidesIdField.set(true); - return; // need to break - } - }); - - if (hasInFilterBesidesIdField.get()) { + // TODO List> + final Optional inFilter = + filters.stream().filter(filter -> filter instanceof DBFilterBase.InFilter).findFirst(); + if (inFilter.isPresent()) { // This if block handles filter with "$in" for non-id field List> expressions = buildConditionExpressions(additionalIdFilter); if (expressions == null) { @@ -604,16 +597,11 @@ private List buildSortedSelectQueries( // that is the reason having this boolean // TODO queryBuilder change for where(Expression> instead of - // List> - AtomicBoolean hasInFilterBesidesIdField = new AtomicBoolean(false); - filters.forEach( - filter -> { - if (filter instanceof DBFilterBase.InFilter) { - hasInFilterBesidesIdField.set(true); - return; // need to break - } - }); - if (hasInFilterBesidesIdField.get()) { + // TODO List> + + final Optional inFilter = + filters.stream().filter(filter -> filter instanceof DBFilterBase.InFilter).findFirst(); + if (inFilter.isPresent()) { // This if block handles filter with "$in" for non-id field List> expressions = buildConditionExpressions(additionalIdFilter); if (expressions == null) {