Major refactoring

Makefile

@@ -24,18 +24,16 @@ OBJS = \
 	src/url.o \
 	src/io.o \
 	src/io_adv.o \
-	src/worker.o \
 	src/offload_tablespace_map.o \
 	src/offload_policy.o \
 	src/offload.o \
 	src/virtual_tablespace.o \
 	src/partition.o \
 	src/xvacuum.o \
-	src/yezzey_expire.o \
 	src/yproxy.o \
 	src/init.o \
 	src/meta.o \
-	smgr.o worker.o yezzey.o
+	smgr.o yezzey.o
 
 EXTENSION = yezzey
 DATA = yezzey--1.0.sql yezzey--1.0--1.8.sql yezzey--1.8--1.8.1.sql

include/gucs.h

@@ -7,11 +7,6 @@ extern int yezzey_ao_log_level;
 extern bool use_gpg_crypto;
 
 /* ----- STORAGE -----  */
-extern char *storage_prefix;
-extern char *storage_bucket;
-extern char *backup_bucket;
-extern char *storage_config;
-extern char *storage_host;
 extern char *storage_class;
 extern int multipart_chunksize;
 extern int multipart_threshold;

include/io_adv.h

@@ -12,12 +12,6 @@ struct IOadv {
   const std::string nspname;
   // relation name itself
   const std::string relname;
-  // s3 host
-  const std::string host;
-  // s3 bucked
-  const std::string bucket;
-  // wal-g specific prefix
-  const std::string external_storage_prefix;
 
   // s3 storage class
   const std::string storage_class;
@@ -39,16 +33,14 @@ struct IOadv {
   const std::string yproxy_socket;
 
   IOadv(const std::string &nspname,
-        const std::string &relname, const std::string &host,
-        const std::string &bucket, const std::string &external_storage_prefix,
+        const std::string &relname,
         const std::string &storage_class, const int &multipart_chunksize,
         /*unparse coords*/ Oid spcNode, const std::string &fileName,
         const Oid reloid, bool use_gpg_crypto,
         const std::string &yproxy_socket);
 
   IOadv(const std::string &nspname,
-        const std::string &relname, const std::string &host,
-        const std::string &bucket, const std::string &external_storage_prefix,
+        const std::string &relname,
         const std::string &storage_class, const int &multipart_chunksize,
         const relnodeCoord &coords, const Oid reloid,
         bool use_gpg_crypto, const std::string &yproxy_socket);

include/url.h

@@ -31,5 +31,4 @@ std::string craftStorageUnPrefixedPath(const std::shared_ptr<IOadv> &adv,
 std::string
 getYezzeyRelationUrl_internal(const std::string &nspname,
                               const std::string &relname,
-                              const std::string &external_storage_prefix,
                               relnodeCoord coords, int32_t segid);

include/util.h

@@ -38,9 +38,6 @@ int64_t yezzey_calc_virtual_relation_size(std::shared_ptr<IOadv> adv,
 std::string storage_url_add_options(const std::string &s3path,
                                     const char *config_path);
 
-std::string getYezzeyExtrenalStorageBucket(const char *host,
-                                           const char *bucket);
-
 std::string make_yezzey_url(const std::string &prefix, int64_t modcounts,
                             XLogRecPtr current_recptr);
 
@@ -49,8 +46,7 @@ std::vector<int64_t> parseModcounts(const std::string &prefix,
 #endif
 
 EXTERNC void getYezzeyExternalStoragePathByCoords(
-    const char *nspname, const char *relname, const char *host,
-    const char *bucket, const char *storage_prefix, Oid spcNode, Oid dbNode,
+    const char *nspname, const char *relname, Oid spcNode, Oid dbNode,
     Oid relNode, int32_t segblockno /* segment no*/, int32_t segid,
     char **dest);
 

include/virtual_index.h

@@ -37,6 +37,8 @@ typedef struct {
   text x_path;           /* external path */
 } FormData_yezzey_virtual_index;
 
+typedef FormData_yezzey_virtual_index* Form_yezzey_virtual_index;
+
 #define Natts_yezzey_virtual_index 10
 #define Anum_yezzey_virtual_index_reloid 1
 #define Anum_yezzey_virtual_index_filenode 2
@@ -57,6 +59,9 @@ EXTERNC void emptyYezzeyIndex(Oid yezzey_index_oid, Oid relfilenode);
 EXTERNC void emptyYezzeyIndexBlkno(Oid yezzey_index_oid, Oid relation,
                                    Oid relfilenode, int blkno);
 
+/* fixup virtual index entry for relation's relfilenode. */
+EXTERNC void YezzeyFixupVirtualIndex(Relation rel);
+
 #ifdef __cplusplus
 void YezzeyVirtualIndexInsert(Oid yandexoid /*yezzey auxiliary index oid*/,
                               Oid reloid, Oid relfilenodeOid, int64_t blkno,

include/yezzey_meta.h

@@ -3,7 +3,6 @@
 #include "pg.h"
 #include "url.h"
 #include "virtual_index.h"
-#include "yezzey_expire.h"
 
 #ifdef __cplusplus
 #define EXTERNC extern "C"

src/init.cpp

@@ -3,10 +3,8 @@
 #include "init.h"
 
 #include "offload_policy.h"
-#include "yezzey_expire.h"
 
 void YezzeyInitMetadata(void) {
   (void)YezzeyCreateOffloadPolicyRelation();
-  (void)YezzeyCreateRelationExpireIndex();
   (void)YezzeyCreateVirtualIndex();
 }

src/io_adv.cpp

@@ -3,16 +3,13 @@
 #include "offload_tablespace_map.h"
 
 IOadv::IOadv(const std::string &nspname,
-             const std::string &relname, const std::string &host,
-             const std::string &bucket,
-             const std::string &external_storage_prefix,
+             const std::string &relname,
              const std::string &storage_class, const int &multipart_chunksize,
              const Oid spcNode, const std::string &fileName, const Oid reloid,
              bool use_gpg_crypto,
              const std::string &yproxy_socket)
     :
-      nspname(nspname), relname(relname), host(host),
-      bucket(bucket), external_storage_prefix(external_storage_prefix),
+      nspname(nspname), relname(relname),
       storage_class(storage_class), multipart_chunksize(multipart_chunksize),
       coords_(getRelnodeCoordinate(spcNode, fileName)), reloid(reloid),
       tableSpace(reloid ? YezzeyGetRelationOriginTablespace(nspname.c_str(), relname.c_str(), reloid) : "none"),
@@ -21,14 +18,11 @@ IOadv::IOadv(const std::string &nspname,
 }
 
 IOadv::IOadv(const std::string &nspname,
-             const std::string &relname, const std::string &host,
-             const std::string &bucket,
-             const std::string &external_storage_prefix,
+             const std::string &relname,
              const std::string &storage_class, const int &multipart_chunksize,
              const relnodeCoord &coords, const Oid reloid,bool use_gpg_crypto,
              const std::string &yproxy_socket)
-    : nspname(nspname), relname(relname), host(host),
-      bucket(bucket), external_storage_prefix(external_storage_prefix),
+    : nspname(nspname), relname(relname),
       storage_class(storage_class), multipart_chunksize(multipart_chunksize),
       coords_(coords), reloid(reloid),
       tableSpace(reloid ? YezzeyGetRelationOriginTablespace(nspname.c_str(), relname.c_str(), reloid) : "none"),

src/meta.cpp

@@ -8,5 +8,5 @@ void YezzeyUpdateMetadataRelations(
   YezzeyVirtualIndexInsert(yandexoid, reloid, relfilenodeOid, blkno,
                            offset_start, offset_finish, encrypted, reused,
                            modcount, lsn, x_path);
-  YezzeyUpsertLastUseLsn(reloid, relfilenodeOid, md5, lsn);
+  /* TODO: update yezzey relfilemap */
 }

src/proxy.cpp

@@ -229,9 +229,6 @@ EXTERNC SMGRFile yezzey_AORelOpenSegFile(Oid reloid, char *nspname,
         if (!RecoveryInProgress()) {
           auto ioadv = std::make_shared<IOadv>(
               yfd.nspname, yfd.relname,
-              std::string(storage_host /*host*/),
-              std::string(storage_bucket /*bucket*/),
-              std::string(storage_prefix /*prefix*/),
               std::string(storage_class /* storage_class */),
               multipart_chunksize, DEFAULTTABLESPACE_OID,
               yfd.filepath /* coords */, reloid /* reloid */,

src/storage.cpp

@@ -196,9 +196,7 @@ int loadSegmentFromExternalStorage(Relation rel, const std::string &nspname,
   /* FIXME */
 
   auto ioadv = std::make_shared<IOadv>(
-      nspname, relname,
-      storage_host /* host */, storage_bucket /*bucket*/,
-      storage_prefix /*prefix*/, storage_class /* storage_class */,
+      nspname, relname, storage_class /* storage_class */,
       multipart_chunksize, coords /* filename */, rel->rd_id /* reloid */,
       use_gpg_crypto, yproxy_socket);
 
@@ -348,9 +346,7 @@ int offloadRelationSegment(Relation aorel, int segno, int64 modcount,
       !external_storage_path ? "" : std::string(external_storage_path);
   ReleaseSysCache(tp);
 
-  auto ioadv = std::make_shared<IOadv>(nspname, relname,
-      storage_host /* host */, storage_bucket /*bucket*/,
-      storage_prefix /*prefix*/, storage_class /* storage_class */,
+  auto ioadv = std::make_shared<IOadv>(nspname, relname, storage_class /* storage_class */,
       multipart_chunksize, coords, aorel->rd_id /* reloid */, use_gpg_crypto, yproxy_socket);
 
   try {
@@ -448,9 +444,6 @@ int statRelationSpaceUsage(Relation aorel, int segno, int64 modcount,
   auto ioadv = std::make_shared<IOadv>(
       nspname,
       std::string(aorel->rd_rel->relname.data),
-      std::string(storage_host /*host*/),
-      std::string(storage_bucket /*bucket*/),
-      std::string(storage_prefix /*prefix*/),
       std::string(storage_class /*storage_class*/), multipart_chunksize,
       coords /* coords */, aorel->rd_id /* reloid */, use_gpg_crypto,
       yproxy_socket);
@@ -512,9 +505,6 @@ int statRelationSpaceUsagePerExternalChunk(Relation aorel, int segno,
   auto ioadv = std::make_shared<IOadv>(
       nspname,
       std::string(aorel->rd_rel->relname.data),
-      std::string(storage_host /*host*/),
-      std::string(storage_bucket /*bucket*/),
-      std::string(storage_prefix /*prefix*/),
       std::string(storage_class /*storage_class*/), multipart_chunksize,
       coords /* coords */, aorel->rd_id /* reloid */, use_gpg_crypto,
       yproxy_socket);

src/url.cpp

@@ -77,8 +77,6 @@ std::string craftStorageUnPrefixedPath(const std::shared_ptr<IOadv> &adv,
 std::string
 getYezzeyRelationUrl_internal(const std::string &nspname,
                               const std::string &relname,
-                              const std::string &external_storage_prefix,
                               relnodeCoord coords, int32_t segid) {
-  return external_storage_prefix +
-         yezzey_block_file_path(nspname, relname, coords, segid);
+  return yezzey_block_file_path(nspname, relname, coords, segid);
 }

src/util.cpp

@@ -23,23 +23,6 @@
 
 const char *baseYezzeyPath = "/basebackups_005/yezzey/";
 
-std::string getYezzeyExtrenalStorageBucket(const char *host,
-                                           const char *bucket) {
-  std::string url = "s3://";
-  std::string toErase = "https://";
-  std::string hostStr = host;
-  size_t pos = hostStr.find(toErase);
-  if (pos != std::string::npos) {
-    hostStr.erase(pos, toErase.length());
-  }
-  url += hostStr;
-  url += "/";
-  url += bucket;
-  url += "/";
-
-  return url;
-}
-
 std::string storage_url_add_options(const std::string &s3path,
                                     const char *config_path) {
   auto ret = s3path;
@@ -87,18 +70,16 @@ relnodeCoord getRelnodeCoordinate(Oid spcNode, const std::string &fileName) {
 }
 
 void getYezzeyExternalStoragePathByCoords(const char *nspname,
-                                          const char *relname, const char *host,
-                                          const char *bucket,
-                                          const char *storage_prefix,
+                                          const char *relname,
                                           Oid spcNode, Oid dbNode, Oid relNode,
                                           int32_t segblockno /* segment no*/,
                                           int32_t segid, char **dest) {
 
   /* FIXME: Support for non-default table space? */
   auto coords = relnodeCoord(spcNode, dbNode, relNode, segblockno);
-  auto prefix = getYezzeyRelationUrl_internal(nspname, relname, storage_prefix,
+  auto prefix = getYezzeyRelationUrl_internal(nspname, relname,
                                               coords, segid);
-  auto path = getYezzeyExtrenalStorageBucket(host, bucket) + prefix;
+  auto path = prefix;
 
   *dest = (char *)malloc(sizeof(char) * path.size());
   strcpy(*dest, path.c_str());

src/virtual_index.cpp

@@ -213,6 +213,58 @@ void emptyYezzeyIndexBlkno(Oid yezzey_index_oid, Oid reloid /* not used */,
   CommandCounterIncrement();
 } /* end emptyYezzeyIndexBlkno */
 
+
+void YezzeyFixupVirtualIndex_internal(Oid yezzey_index_oid, Relation relation) {
+
+  HeapTuple tuple;
+  ScanKeyData skey[1];
+  bool nulls[Natts_yezzey_virtual_index];
+  Datum values[Natts_yezzey_virtual_index];
+
+  memset(nulls, 0, sizeof(nulls));
+  memset(values, 0, sizeof(values));
+
+  auto rel = heap_open(yezzey_index_oid, RowExclusiveLock);
+
+  auto snap = RegisterSnapshot(GetTransactionSnapshot());
+
+  ScanKeyInit(&skey[0], Anum_yezzey_virtual_index_filenode,
+              BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relation->rd_node.relNode));
+
+  auto desc = yezzey_beginscan(rel, snap, YezzeyVirtualIndexScanCols, skey);
+
+  while (HeapTupleIsValid(tuple = heap_getnext(desc, ForwardScanDirection))) {
+        // update
+    auto meta = (Form_yezzey_virtual_index)GETSTRUCT(tuple);
+
+    // Assert(meta->yrelfileoid == relfileoid);
+
+    values[Anum_yezzey_virtual_index_filenode - 1] =
+        ObjectIdGetDatum(relation->rd_node.relNode);
+
+    auto yandxtuple = heap_form_tuple(RelationGetDescr(relation), values, nulls);
+
+#if IsGreenplum6
+    simple_heap_update(relation, &tuple->t_self, yandxtuple);
+    CatalogUpdateIndexes(relation, yandxtuple);
+#else
+    CatalogTupleUpdate(relation, &tuple->t_self, yandxtuple);
+#endif
+  }
+
+  yezzey_endscan(desc);
+  heap_close(rel, RowExclusiveLock);
+
+  UnregisterSnapshot(snap);
+
+  /* make changes visible*/
+  CommandCounterIncrement();
+} 
+
+void YezzeyFixupVirtualIndex(Relation relation) {
+  (void) YezzeyFixupVirtualIndex_internal(YezzeyFindAuxIndex(RelationGetRelid(relation)), relation);
+} /* end YezzeyFixupVirtualIndex */
+
 void YezzeyVirtualIndexInsert(Oid yandexoid /*yezzey auxiliary index oid*/,
                               Oid reloid, Oid relfilenodeOid, int64_t blkno,
                               int64_t offset_start, int64_t offset_finish,