Allow processing of duplicated attribute data columns (#203)

Ensure the data in a duplicated column is returned only once (and not e.g. twice if 2 columns with the same name are present) during processing.

xl2times/transforms.py

@@ -468,6 +468,7 @@ def process_flexible_import_table(
         known_columns = config.known_columns[datatypes.Tag.fi_t]
         data_columns = [x for x in df.columns if x not in known_columns]
 
+        # TODO: Replace this with something similar to know columns from config
         # Populate index columns
         index_columns = [
             "region",

xl2times/utils.py

@@ -60,7 +60,14 @@ def explode(df, data_columns):
     :return:                Tuple with the exploded dataframe and a Series of the original
                             column name for each value in each new row.
     """
-    data = df[data_columns].values.tolist()
+    # Handle duplicate columns (https://pandas.pydata.org/docs/user_guide/duplicates.html)
+    if len(set(data_columns)) < len(data_columns):
+        cols = df.columns.to_list()
+        data_cols_idx = [idx for idx, val in enumerate(cols) if val in data_columns]
+        data = df.iloc[:, data_cols_idx].values.tolist()
+    else:
+        data = df[data_columns].values.tolist()
+
     other_columns = [
         colname for colname in df.columns.values if colname not in data_columns
     ]
@@ -69,7 +76,6 @@ def explode(df, data_columns):
     df = df.assign(value=data)
     nrows = df.shape[0]
     df = df.explode(value_column, ignore_index=True)
-
     names = pd.Series(data_columns * nrows, index=df.index, dtype=str)
     # Remove rows with no VALUE
     index = df[value_column].notna()