From 630f41edaa79fb0c752e3eec03dae5d1b73896ef Mon Sep 17 00:00:00 2001 From: Sharad Chandran R Date: Tue, 23 Jul 2024 17:05:39 +0530 Subject: [PATCH] Update Documentation for Binary Vectors and BFILE support --- doc/src/api_manual/connection.rst | 6 +- doc/src/api_manual/lob.rst | 86 +++++++++++ doc/src/api_manual/oracledb.rst | 12 ++ doc/src/user_guide/appendix_a.rst | 8 +- doc/src/user_guide/lob_data.rst | 165 +++++++++++++++++++-- doc/src/user_guide/vector_data_type.rst | 181 ++++++++++++++++++++---- 6 files changed, 417 insertions(+), 41 deletions(-) diff --git a/doc/src/api_manual/connection.rst b/doc/src/api_manual/connection.rst index 2b94aef2b..99ac5c137 100644 --- a/doc/src/api_manual/connection.rst +++ b/doc/src/api_manual/connection.rst @@ -611,8 +611,8 @@ Connection Methods LOBs created with ``createLob()`` can be bound for IN, IN OUT and OUT binds. - See :ref:`Working with CLOB, NCLOB and BLOB Data ` and :ref:`LOB - Bind Parameters ` for more information. + See :ref:`Working with CLOB, NCLOB, BLOB and BFILE Data ` and + :ref:`LOB Bind Parameters ` for more information. The parameters of the ``connection.createLob()`` method are: @@ -632,7 +632,7 @@ Connection Methods - Description * - ``type`` - Number - - One of the constants :ref:`oracledb.CLOB `, :ref:`oracledb.BLOB `, or :ref:`oracledb.NCLOB ` (or equivalent ``DB_TYPE_*`` constants). + - One of the constants :ref:`oracledb.CLOB `, :ref:`oracledb.BLOB `, :ref:`oracledb.NCLOB `, or :ref:`oracledb.BFILE ` (or equivalent ``DB_TYPE_*`` constants). **Callback**: diff --git a/doc/src/api_manual/lob.rst b/doc/src/api_manual/lob.rst index e1b502152..754d444f2 100644 --- a/doc/src/api_manual/lob.rst +++ b/doc/src/api_manual/lob.rst @@ -155,6 +155,46 @@ Lob Methods * - Error ``error`` - This optional parameter is used for the error emitted in an ``error`` event. +.. method:: lob.fileExists() + + .. versionadded:: 6.6 + + **Promise**:: + + promise = fileExists(); + + Returns a boolean which indicates whether the file specified by the LOB + exists in the directory alias or not. This method returns *true* if the + file exists in the directory and *false* if it does not. If the directory + that this method is trying to access does not exist, then this method + returns an error. This method can only be used if the LOB type is BFILE. + For all other LOB types, this method returns the ``NJS-156: operation is + only supported on BFILE LOBs`` error. + + **Callback**: + + If you are using the callback programming style:: + + fileExists(function(Error error, Boolean val)); + + The parameters of the callback function + ``function(Error error, Boolean val)`` are: + + .. list-table-with-summary:: + :header-rows: 1 + :class: wy-table-responsive + :align: center + :widths: 15 30 + :summary: The first column displays the callback function parameter. + The second column displays the description of the parameter. + + * - Callback Function Parameter + - Description + * - Error ``error`` + - If ``fileExists()`` succeeds, ``error`` is NULL. If an error occurs, then ``error`` contains the :ref:`error message `. + * - Boolean ``val`` + - Indicates whether the file exists in the directory. This parameter will be *true* if the file exists in the directory and *false* if it does not. + .. method:: lob.getData() .. versionadded:: 4.0 @@ -241,3 +281,49 @@ Lob Methods - If ``getData()`` succeeds, ``error`` is NULL. If an error occurs, then ``error`` contains the :ref:`error message `. * - String ``data`` or Buffer ``data`` - The value of the LOB. + +.. method:: lob.getDirFileName() + + .. versionadded:: 6.6 + + .. code-block:: javascript + + getDirFileName(); + + This synchronous method returns an object containing the directory alias + and file name of the LOB object. This method can only be used if the LOB + type is BFILE. For all other LOB types, this method returns the + ``NJS-156: operation is only supported on BFILE LOBs`` error. + +.. method:: lob.setDirFileName() + + .. versionadded:: 6.6 + + .. code-block:: javascript + + setDirFileName(Object dirFileName); + + This synchronous method sets the directory alias and file name of the LOB. + This method can only be used if the LOB type is BFILE. For all other LOB + types, this method returns the ``NJS-156: operation is only supported on + BFILE LOBs`` error. + + .. note:: + + This method can only be used in node-oracledb Thin mode. + + The parameters of the ``lob.setDirFileName()`` method are: + + .. list-table-with-summary:: lob.setDirFileName() Parameters + :header-rows: 1 + :class: wy-table-responsive + :align: center + :widths: 10 10 30 + :summary: The first column displays the parameter. The second column displays the data type of the parameter. The third column displays the description of the parameter. + + * - Parameter + - Data Type + - Description + * - ``dirFileName`` + - Object + - This parameter contains the directory alias and file name of the BFILE type LOB. diff --git a/doc/src/api_manual/oracledb.rst b/doc/src/api_manual/oracledb.rst index 0c5cfd484..d0a1b2112 100644 --- a/doc/src/api_manual/oracledb.rst +++ b/doc/src/api_manual/oracledb.rst @@ -214,6 +214,10 @@ for common :ref:`Oracle Database Type Constants `. - Value - ``DbType`` Object Equivalent - Notes + * - ``oracledb.BFILE`` + - 2020 + - ``oracledb.DB_TYPE_BFILE`` + - * - ``oracledb.BLOB`` - 2019 - ``oracledb.DB_TYPE_BLOB`` @@ -874,6 +878,7 @@ Constants for two-phase commit (TPC) functions Vector Type Constants --------------------- + .. versionadded:: 6.5 Constants for the :ref:`vectorFormat ` attribute. @@ -899,6 +904,13 @@ Constants for the :ref:`vectorFormat ` attribute. * - ``oracledb.VECTOR_FORMAT_INT8`` - 4 - The storage format of each dimension value in the VECTOR column is an 8-bit signed integer. + * - ``oracledb.VECTOR_FORMAT_BINARY`` + - 5 + - The storage format of each dimension value in the VECTOR column is represented as a single bit. All the dimensions for the vector are stored as an array of 8-bit unsigned integers. + +.. versionchanged:: 6.6 + + The ``oracledb.VECTOR_FORMAT_BINARY`` constant was added. .. _oracledbproperties: diff --git a/doc/src/user_guide/appendix_a.rst b/doc/src/user_guide/appendix_a.rst index b5d0503ae..8a466d68e 100644 --- a/doc/src/user_guide/appendix_a.rst +++ b/doc/src/user_guide/appendix_a.rst @@ -156,7 +156,7 @@ node-oracledb Thin and Thick modes. For more details see :ref:`modediff`. - No - Yes * - SQL execution (see :ref:`sqlexecution`) - - Yes - Bind and fetch all types except BFILE + - Yes - Yes * - PL/SQL execution (see :ref:`plsqlexecution`) - Yes - For scalar types and collection types using array interface @@ -297,8 +297,8 @@ node-oracledb Thin and Thick modes. For more details see :ref:`modediff`. - Yes - Yes - May need to fetch as CLOB * - BFILE data type - - No - - No + - Yes + - Yes * - TIMESTAMP WITH TIME ZONE data type (see :ref:`oracledbconstantsdbtype`) - Yes - Yes @@ -734,7 +734,7 @@ when binding numeric values. - Yes * - BFILE - DB_TYPE_BFILE - - No + - Yes * - BLOB - DB_TYPE_BLOB - Yes diff --git a/doc/src/user_guide/lob_data.rst b/doc/src/user_guide/lob_data.rst index 6565dd411..39fe524b9 100644 --- a/doc/src/user_guide/lob_data.rst +++ b/doc/src/user_guide/lob_data.rst @@ -1,16 +1,26 @@ .. _lobhandling: -******************************* -Using CLOB, NCLOB and BLOB Data -******************************* +*************************************** +Using CLOB, NCLOB, BLOB, and BFILE Data +*************************************** Oracle Database uses LOB data types to store long objects. The CLOB type is used for character data and the BLOB type is used for binary data. NCLOB can hold character data in the database’s alternative `national character set `__. -In node-oracledb, LOBs can be represented by instances of the -:ref:`Lob ` class or as Strings and Buffers. +-AA8D783D-7337-4A61-BD7D-5DB580C46D9A>`__. The `BFILE `__ +type is used for referencing a file stored on the server operating system +where Oracle Database is running, outside the database tablespace. The BFILE +LOB type was introduced in node-oracledb 6.6. + +node-oracledb uses :ref:`oracledb.CLOB `, +:ref:`oracledb.BLOB `, +:ref:`oracledb.NCLOB `, and +:ref:`oracledb.BFILE ` to represent CLOB, BLOB, +NCLOB, and BFILE data types respectively. In node-oracledb, LOBs can be +represented by instances of the :ref:`Lob ` class or as Strings and +Buffers. There are runnable LOB examples in the GitHub `examples `__ @@ -25,6 +35,11 @@ Node.js String or Buffer types can be passed into PL/SQL blocks or inserted into the database by binding to LOB columns or PL/SQL parameters. +.. _insertclobblob: + +Inserting CLOBs and BLOBs +------------------------- + Given the table: .. code-block:: sql @@ -99,13 +114,105 @@ Node.js or node-oracledb, it will need to be streamed to a :ref:`Lob `, as discussed in :ref:`Streaming Lobs `. +See :ref:`fetchinglob` for information on how to fetch CLOBs, BLOBs, and NCLOBs. + +.. _insertbfile: + +Inserting BFILEs +---------------- + +The data of `BFILE `__ type LOB is stored as files in a +directory in the Oracle Database server. The column of type BFILE stores a +reference to the file stored in the Oracle Database server file system. +The BFILE column data cannot be updated from within your application since +Oracle Database allows read-only access to the data stored in BFILE columns. + +Before using the BFILE data type, ensure that you have created a directory in +the database server file system to store the file. Each BFILE object is +associated with: + +- A DIRECTORY object which is an alias for the directory on the database + server file system that stores the file with BFILE data. For example, if + your server is running on a Linux machine, you can create a DIRECTORY object + by using: + + .. code-block:: sql + + CREATE OR REPLACE DIRECTORY MYBFILEDIR AS '/tmp/my-bfile-dir' + + ``MYBFILEDIR`` is the directory alias. + ``/tmp/my-bfile-dir`` is the physical operating system directory in the + database server file system. It is a string containing the full path name of + the directory and follows the operating system rules. + + This directory and alias are used in subsequent examples. +- The file name of the physical file which is stored in the directory in the + database server file system. For example, ``MYBFILE.JPG``. The file in this + directory can be copied using operating system commands such as ``cp`` or + ``COPY``. This file name is used in subsequent examples. + +Ensure that you have the required access permissions to the directory. For +Windows platform, ensure that you have set the Access Control Lists (ACL) or +Discretionary Access Control List (DACL) correctly to access the file. + +The following table will be used in the subsequent examples to demonstrate +using ``BFILE`` data with node-oracledb: + +.. code-block:: sql + + CREATE TABLE bfile_table( + id NUMBER, + bfilecol BFILE + ); + +To insert data of BFILE data type: + +.. code-block:: javascript + + const result = await connection.execute( + `INSERT INTO bfile_table VALUES (:id, BFILENAME(:BFILEDIR, :BFILENAME))`, + [101, "MYBFILEDIR", "MYBFILE.JPG"]); + +This example inserts a row in ``bfile_table`` with BFILE properties, +directory alias ``MYBFILEDIR`` and file name ``MYBFILE.JPG``. + +Note that the content in the BFILE column cannot be updated. You can only +update the properties such as directory alias and the file name. Once updated, +the LOB object references the new file name specified. To update the file name +to ``NEWBFILE.JPG``, you can use: + +.. code-block:: javascript + + const result = await connection.execute( + `UPDATE bfile_table SET bfilecol = BFILENAME("MYBFILEDIR", "NEWBFILE.JPG") WHERE id = :ID`, + [101]); + +In node-oracledb Thin mode, you can set the directory alias and file name +using :meth:`lob.setDirFileName()`. For example: + +.. code-block:: javascript + + const result = await conn.execute(` + SELECT bfilecol FROM bfile_table WHERE id = :id`, [101]); + const lob = result.rows[0][0]; + const dirFile = lob.getDirFileName(); + lob.setDirFileName({dirName: "NEWALIASNAME", fileName: "NEWBFILENAME"}); + +This will update the directory alias to ``NEWALIASNAME`` and file name to +``NEWBFILENAME``. + +See :ref:`fetchbfile` for information on how to query a BFILE column. + .. _queryinglobs: Simple LOB Queries and PL/SQL OUT Binds ======================================= -Querying LOBs -------------- +.. _fetchinglob: + +Fetching CLOBs, BLOBs, and NCLOBs +--------------------------------- LOBs queried from the database that are shorter than 1 GB can be returned as Strings or Buffers by using @@ -219,6 +326,48 @@ in Node.js or node-oracledb, it will need to be explicitly streamed to a Lobs `. See :ref:`LOB Bind Parameters ` for size considerations regarding LOB binds. +.. _fetchbfile: + +Fetching BFILEs +--------------- + +To query the BFILE column of ``bfile_table``, you can use: + +.. code-block:: javascript + + const result = await connection.execute( + `SELECT bfilecol FROM bfile_table WHERE id = :id`, [101]); + const lob = result.rows[0][0]; + const dirFile = lob.getDirFileName(); + console.log("Directory Alias:", dirFile.dirName, "File Name:", dirFile.fileName); + +This prints the following output:: + + MYBFILEDIR, MYBFILE.JPG + +To query the metadata of a BFILE column, you can use: + +.. code-block:: javascript + + const result = await connection.execute(`SELECT bfilecol FROM bfile_table`); + console.log("Metadata:", result.metaData); + +This query prints the metadata for the ``bfilecol`` column and displays the +dbType as ``DB_TYPE_BFILE``:: + + MetaData: [ + { + name: 'BFILECOL', + dbType: [DbType DB_TYPE_BFILE], + nullable: false, + isJson: false, + isOson: false, + dbTypeName: 'BFILE', + fetchType: [DbType DB_TYPE_BFILE], + converter: [Function: converter] + }, + ] + .. _streamsandlobs: Streaming Lobs diff --git a/doc/src/user_guide/vector_data_type.rst b/doc/src/user_guide/vector_data_type.rst index e6e18a9bd..0d0b5d8eb 100644 --- a/doc/src/user_guide/vector_data_type.rst +++ b/doc/src/user_guide/vector_data_type.rst @@ -1,15 +1,36 @@ .. _vectors: ***************** -Using Vector Data +Using VECTOR Data ***************** -Oracle Database 23ai introduced a new data type VECTOR for artificial -intelligence and machine learning search operations. The vector data type -is a homogeneous array of 8-bit signed integers, 32-bit floating-point -numbers, or 64-bit floating-point numbers. With the vector data type, you -can define the number of dimensions for the data and the storage format -for each dimension value in the vector. +Oracle Database 23ai introduced a new data type `VECTOR `__ for artificial intelligence and machine learning search operations. +The VECTOR data type is a homogeneous array of 8-bit signed integers, 8-bit +unsigned integers, 32-bit floating-point numbers, or 64-bit floating-point +numbers. + +With the VECTOR data type, you can define the number of dimensions for the +data and the storage format for each dimension value in the vector. The +possible storage formats include: + +- **int8** for 8-bit signed integers +- **binary** for 8-bit unsigned integers +- **float32** for 32-bit floating-point numbers +- **float64** for 64-bit floating point numbers + +You can also create vector columns without specifying the number of dimensions +or their storage format. For example: + +.. code-block:: sql + + CREATE TABLE vecTable ( + dataVec VECTOR + ) + +Using FLOAT32, FLOAT64, and INT8 Vectors +======================================== To create a table with three columns for vector data, for example: @@ -25,22 +46,14 @@ In this example, each column can store vector data of three dimensions where each dimension value is of the specified storage format. This example is used in subsequent sections. -You can also create vector columns without specifying the number of dimensions -or their storage format. For example: - -.. code-block:: sql - - CREATE TABLE vecTable ( - dataVec VECTOR - ) - .. _insertvector: -Inserting Vectors -================= +Inserting FLOAT32, FLOAT64, and INT8 Vectors +-------------------------------------------- -With node-oracledb, vector data can be inserted using TypedArrays as bind -values. +With node-oracledb, vector data of types INT8, FLOAT32, and FLOAT64 can be +inserted using TypedArrays or JavaScript arrays as bind values. To insert +using TypedArrays: .. code-block:: javascript @@ -59,9 +72,10 @@ values. { vec32: float32arr, vec64: float64arr, vec8: int8arr } ); -To insert TypeArrays in vector columns that are defined without a specific +To insert data in vector columns that are defined without a specific dimension or storage format, you should set the ``type`` attribute to -``oracledb.DB_TYPE_VECTOR`` as shown below: +``oracledb.DB_TYPE_VECTOR``. The following example uses a JavaScript +array to insert vector data: .. code-block:: javascript @@ -75,8 +89,8 @@ dimension or storage format, you should set the ``type`` attribute to .. _fetchvector: -Fetching Vectors -================ +Fetching FLOAT32, FLOAT64, and INT8 Vectors +------------------------------------------- With node-oracledb, vector columns are fetched as TypedArrays of signed integer (8-bit), float (32-bit), or double (64-bit) depending on whether the @@ -136,7 +150,7 @@ these attributes, you can use: console.log('Vector dimensions for the VCOL32 column:', vecDimensions); console.log('Vector storage format for the VCOL32 column:', vecStorageFormat); -This prints an ouput such as:: +This prints the following output:: Vector dimensions for the VCOL32 column: 3 Vector storage format for the VCOL32 column: 2 @@ -144,9 +158,11 @@ This prints an ouput such as:: This output indicates that the ``VCOL32`` column in vecTable is a 3-dimensional FLOAT32 vector. +.. _fetchtypehandlervector: + Using a :ref:`fetch type handler `, you can convert the vector data that was fetched to a JavaScript array, if required. Consider the -following example which converts a TypedArray to a Javascript array. +following example which converts a TypedArray to a JavaScript array. .. code-block:: javascript @@ -188,3 +204,116 @@ See `vectortype1.js `__ and `vectortype2.js `__ for runnable examples. + +.. _binaryvectors: + +Using BINARY Vectors +==================== + +In addition to INT8, FLOAT32, and FLOAT64 formats, you can also use a +BINARY format to define vectors. The BINARY format represents each dimension +value as a binary value (0 or 1). Binary vectors require less memory storage. +For example, a 16 dimensional vector with BINARY format requires only 2 bytes of +storage while a 16 dimensional vector with INT8 format requires 16 bytes of +storage. The BINARY vector support was introduced in node-oracledb 6.6. + +Binary vectors are represented as 8-bit unsigned integers. For the BINARY +format, you must define the number of dimensions as a multiple of 8. To create +a table with one column for vector data: + +.. code-block:: sql + + CREATE TABLE vecBinaryTable ( + VCOLB VECTOR(16, BINARY) + ) + +In this example, the ``VCOLB`` column can store vector data of 16 dimensions +where each dimension value is represented as a single bit. Note that the +number of dimensions 16 is a multiple of 8. This example is used in the +subsequent sections. + +If you specify a vector dimension that is not a multiple of 8, then you will +get an error. + +.. _insertbinaryvector: + +Inserting BINARY Vectors +------------------------ + +With node-oracledb, vector data of type binary can be inserted using +TypedArrays as bind values. The length of 8-bit unsigner integer arrays must +be equal to the number of dimensions divided by 8. For example, if the number +of dimensions for a vector column is 24, then the length of the array must be +3. The values in these arrays can range from 0 to 255. For example: + +.. code-block:: javascript + + // 8-bit unsigned integer TypedArray for representing binary vectors + const uInt8Arr = new Uint8Array([240, 200]); + + await connection.execute( + `INSERT INTO vecBinaryTable (VCOLB) VALUES (:vecb)`, + { vecb: uInt8Arr } + ); + +In the above example, the length of ``uInt8Arr`` is 2 since the number of +dimensions defined for the vector column ``VCOLB`` is 16. + +Vector data of format BINARY cannot be inserted using JavaScript arrays as +bind values and will return an error. + +.. _fetchbinaryvector: + +Fetching BINARY Vectors +----------------------- + +With node-oracledb, vector columns are fetched as TypedArrays of unsigned +integers (8-bit) if the VECTOR column in Oracle Database contains +BINARY data. To query a VECTOR column: + +.. code-block:: javascript + + const result = await connection.execute( + `SELECT VCOLB FROM vecBinaryTable` + ); + const vecb = result.rows[0].VCOLB; + console.log('Returned Array Type:', vecb.constructor); + console.log('Returned Array:', vecb); + +This prints an output such as:: + + Returned Array type: [Function: Uint8Array] + Returned Array: Uint8Array(2) [ + 240, + 200 + ] + +The :ref:`vectorDimensions ` and +:ref:`vectorFormat ` attributes in the metadata returned by a +query contains the number of dimensions of the vector column and the storage +format of each dimension value in the vector column respectively. To fetch +these attributes, you can use: + +.. code-block:: javascript + + const vecDimensions = result.metadata[0].vectorDimensions; + const vecStorageFormat = result.metadata[0].vectorFormat; + console.log('Vector dimensions for the VCOLB column:', vecDimensions); + console.log('Vector storage format for the VCOLB column:', vecStorageFormat); + +This prints the following output:: + + Vector dimensions for the VCOLB column: 16 + Vector storage format for the VCOLB column: 5 + +This output indicates that the ``VCOLB`` column in vecBinaryTable is a +16-dimensional BINARY vector. + +Using the fetch type handler shown in this +:ref:`section `, you can convert the vector data that +was fetched to a JavaScript array. + +See `vectortype1.js `__ and `vectortype2.js `__ for runnable +examples.