updated on 20200702 at 17:43:18

processing/species/README.md

@@ -10,7 +10,6 @@ Since dataset is made by different sources (IUCN and Birdlife) and data models (
 +  Some **selected attributes** are extracted and transformed from IUCN **geometric** data:
    +  id_no (bigint),
    +  binomial (text),
-   +  legend (text),
    +  kingdom (text),
    +  phylum (text),
    +  class (text),
@@ -24,6 +23,12 @@ Since dataset is made by different sources (IUCN and Birdlife) and data models (
 
 +  Birdlife **geometric** data are processed, and **selected attributes** are extracted, in the way to get the **same structure** of processed IUCN data. 
 
+    Code is: [creates_attributes_sp_birdlife.sql](./species_2020/creates_attributes_sp_birdlife.sql), output table is: **species_202001.attributes_sp_birdlife**;
+
++  IUCN and Birdlife **selected attributes from geometric** data are appended each other.
+
+    Code is: [creates_attributes_sp.sql](./species_2020/creates_attributes_sp.sql), output table is: **species_202001.attributes_sp**;
+
 +  not all the geometric information is used (geometries are imported only WHERE presence IN (1,2) AND origin IN (1,2) AND seasonal IN (1,2,3); which include: Extant and Probably Extant; Native and Reintroduced; Resident, Breeding Season and Non-breeding Season. Therefore, only correspondant species are included from non-spatial dataset.
 
 +  The flattening workflow get rid of:

processing/species/species_2020/creates_attributes_sp.sql

@@ -0,0 +1,5 @@
+DROP TABLE IF EXISTS species_202001.attributes_sp;CREATE TABLE species_202001.attributes_sp AS
+SELECT * FROM species_202001.attributes_sp_birdlife
+UNION
+SELECT * FROM species_202001.attributes_sp_iucn
+ORDER BY class,order_,family,genus,id_no;

processing/species/species_2020/creates_attributes_sp_birdlife.sql

@@ -0,0 +1,24 @@
+-- creates table BIRDLIFE_SPATIAL_ATTRIBUTES as selection of attributes from spatial data
+DROP TABLE IF EXISTS species_202001.attributes_sp_birdlife; 
+CREATE TABLE species_202001.attributes_sp_birdlife AS
+WITH 
+a AS (
+SELECT DISTINCT
+id_no::bigint,
+binomial::text,
+kingdom::text,
+phylum::text,
+class::text,
+order_::text,
+family::text,
+genus::text,
+category::text,
+-- aggregates biome_marine,biome_terrestrial and biome_freshwater ecosystems in ecosystem_mtf (marine,terrestrial,freshwater).
+CONCAT(
+(CASE WHEN biome_marine = 'true' THEN 1 ELSE 0 END),
+(CASE WHEN biome_terrestrial = 'true' THEN 1 ELSE 0 END),
+(CASE WHEN biome_freshwater = 'true' THEN 1 ELSE 0 END)) ecosystem_mtf
+FROM species_202001.birds_additional_atts
+ORDER BY id_no)
+-- select only attributes matching geometric objects
+SELECT DISTINCT * FROM a WHERE id_no IN (SELECT DISTINCT sisid FROM species_202001.birds_all_species) ORDER BY id_no;

processing/species/species_2020/creates_attributes_sp_iucn.sql

@@ -4,7 +4,6 @@ CREATE TABLE species_202001.attributes_sp_iucn AS
 SELECT DISTINCT
 id_no,
 binomial::text,
-legend::text,
 kingdom::text,
 phylum::text,
 class::text,

processing/species/species_2020/fix_missing_ecosystems.sql

@@ -7,7 +7,28 @@ WHERE marine IS NULL OR freshwater IS NULL OR terrestrial IS NULL)
 -- find, in IUCN table "assessments", missing ecosystems for those taxa that lack this information in the BIRDLIFE table "taxonomic"
 SELECT internaltaxonid,systems FROM species_202001.assessments JOIN missing_ecosystems ON internaltaxonid::integer = sisrecid;
 
--- fill, in the BIRDLIFE table "taxonomic", the missing ecosystems (they are all terrestrial).
+-- Add, in the BIRDLIFE table "additional_atts", the missing information related to missing ecosystems
+-- IMPORTANT! THIS HAS BEEN USED TO FILL THE FINAL TABLE
+DROP TABLE IF EXISTS missing;CREATE TEMPORARY TABLE missing AS SELECT sisrecid,marine,freshwater,terrestrial FROM species_birdlife_201903.birdlife_hbw_taxonomic_checklist_v4 ORDER BY sisrecid;
+DELETE FROM missing WHERE marine IS NOT NULL OR freshwater IS NOT NULL OR terrestrial IS NOT NULL;
+INSERT INTO species_202001.birds_additional_atts(id_no,binomial,kingdom,phylum,class,order_,family,genus,category,biome_marine,biome_terrestrial,biome_freshwater)
+SELECT DISTINCT
+a.sisrecid id_no,
+a.scientific_name AS binomial,
+'ANIMALIA' AS kingdom,
+'CHORDATA' AS phylum,
+'AVES' AS class,
+a.order_,
+UPPER(a.family_name) AS "family",
+SPLIT_PART(a.scientific_name, ' ', 1) AS genus,
+a.f2019_iucn_red_list_category AS category,
+'false'::text AS biome_marine,
+'true'::text AS biome_terrestrial,
+'false'::text AS biome_freshwater
+FROM species_202001.birds_taxonomic a JOIN missing b USING(sisrecid);
+
+-- fill, in the BIRDLIFE table "taxonomic", the missing ecosystems (they are all terrestrial)
+-- Useful, but this has not been used for filling the final table
 WITH
 missing_ecosystems AS
 (SELECT DISTINCT sisrecid,'F' marine,'F' freshwater,'T'terrestrial
@@ -20,3 +41,5 @@ freshwater=b.freshwater,
 terrestrial=b.terrestrial
 FROM missing_ecosystems b
 WHERE birds_taxonomic.sisrecid=b.sisrecid;
+
+

processing/species/species_2020/new_views.sql

@@ -1,102 +1,3 @@
-WITH birds_geoms AS (
-WITH
-birds_geoms AS (
-SELECT
-DISTINCT sisid::bigint AS id_no,
-sciname AS binomial,
-presence,
-origin,
-seasonal
-FROM species_202001.birds_all_species birds_geom
-),
-birds_taxonomic AS (
-SELECT DISTINCT
-sisrecid::bigint AS id_no,
-order_,
-family_name,
-family,
-subfamily_name,
-tribe_name,
-common_name,
-scientific_name,
-f2019_iucn_red_list_category,
-marine,
-freshwater,
-terrestrial
-FROM species_202001.birds_taxonomic
-),
-birds_additional_atts AS (
-SELECT DISTINCT
-id_no,
-binomial,
-common_name,
-kingdom,
-phylum,
-class,
-order_,
-family,
-genus,
-category,
-biome_marine AS marine,
-biome_terrestrial AS terrestrial,
-biome_freshwater AS freshwater
-FROM (species_202001.birds_additional_atts
- JOIN birds_geoms USING (id_no, binomial))
-),
-missing_species AS (
-SELECT DISTINCT id_no
-FROM birds_geoms
-WHERE (NOT (id_no IN ( SELECT DISTINCT id_no
-	   FROM birds_additional_atts)))
-), missing_systems AS (
-SELECT id_no,
-scientific_name AS binomial,
-common_name,
-'ANIMALIA'::character varying AS kingdom,
-'CHORDATA'::character varying AS phylum,
-'AVES'::character varying AS class,
-order_,
-upper((family_name)::text) AS family,
-(split_part((scientific_name)::text, ' '::text, 1))::character varying AS genus,
-f2019_iucn_red_list_category AS category,
-'false'::text AS marine,
-'true'::text AS terrestrial,
-'false'::text AS freshwater
-FROM birds_taxonomic
-WHERE (id_no IN ( SELECT missing_species.id_no
-	   FROM missing_species))
-)
-SELECT id_no,
-binomial,
-common_name,
-kingdom,
-phylum,
-class,
-order_,
-family,
-genus,
-category,
-marine,
-terrestrial,
-freshwater
-FROM birds_additional_atts
-UNION
-SELECT missing_systems.id_no,
-missing_systems.binomial,
-missing_systems.common_name,
-missing_systems.kingdom,
-missing_systems.phylum,
-missing_systems.class,
-missing_systems.order_,
-missing_systems.family,
-missing_systems.genus,
-missing_systems.category,
-missing_systems.marine,
-missing_systems.terrestrial,
-missing_systems.freshwater
-FROM missing_systems
-ORDER BY 1;
-
 CREATE VIEW species.v_mt_categories AS
  SELECT
         CASE a.redlistcategory
@@ -402,149 +303,3 @@ CREATE VIEW species.v_mt_usetrade AS
    FROM species.additional_tables_usetrade
   ORDER BY (additional_tables_usetrade.code)::integer;
 
-CREATE VIEW species.v_redlist_all AS
- WITH corals AS (
-         SELECT DISTINCT corals.id_no,
-            corals.binomial,
-            corals.presence,
-            corals.origin,
-            corals.seasonal,
-            corals.legend,
-            corals.kingdom,
-            corals.phylum,
-            corals.class,
-            corals.order_,
-            corals.family,
-            corals.genus,
-            corals.category,
-            corals.marine,
-            corals.terrestial,
-            corals.freshwater
-           FROM species.corals
-        ), chondrichthyes AS (
-         SELECT DISTINCT chondrichthyes.id_no,
-            chondrichthyes.binomial,
-            chondrichthyes.presence,
-            chondrichthyes.origin,
-            chondrichthyes.seasonal,
-            chondrichthyes.legend,
-            chondrichthyes.kingdom,
-            chondrichthyes.phylum,
-            chondrichthyes.class,
-            chondrichthyes.order_,
-            chondrichthyes.family,
-            chondrichthyes.genus,
-            chondrichthyes.category,
-            chondrichthyes.marine,
-            chondrichthyes.terrestial,
-            chondrichthyes.freshwater
-           FROM species.chondrichthyes
-        ), amphibians AS (
-         SELECT DISTINCT amphibians.id_no,
-            amphibians.binomial,
-            amphibians.presence,
-            amphibians.origin,
-            amphibians.seasonal,
-            amphibians.legend,
-            amphibians.kingdom,
-            amphibians.phylum,
-            amphibians.class,
-            amphibians.order_,
-            amphibians.family,
-            amphibians.genus,
-            amphibians.category,
-            amphibians.marine,
-            amphibians.terrestial,
-            amphibians.freshwater
-           FROM species.amphibians
-        ), mammals AS (
-         SELECT DISTINCT mammals.id_no,
-            mammals.binomial,
-            mammals.presence,
-            mammals.origin,
-            mammals.seasonal,
-            mammals.legend,
-            mammals.kingdom,
-            mammals.phylum,
-            mammals.class,
-            mammals.order_,
-            mammals.family,
-            mammals.genus,
-            mammals.category,
-            mammals.marine,
-            mammals.terrestial,
-            mammals.freshwater
-           FROM species.mammals
-        )
- SELECT corals.id_no,
-    corals.binomial,
-    corals.presence,
-    corals.origin,
-    corals.seasonal,
-    corals.legend,
-    corals.kingdom,
-    corals.phylum,
-    corals.class,
-    corals.order_,
-    corals.family,
-    corals.genus,
-    corals.category,
-    corals.marine,
-    corals.terrestial,
-    corals.freshwater
-   FROM corals
-UNION
- SELECT chondrichthyes.id_no,
-    chondrichthyes.binomial,
-    chondrichthyes.presence,
-    chondrichthyes.origin,
-    chondrichthyes.seasonal,
-    chondrichthyes.legend,
-    chondrichthyes.kingdom,
-    chondrichthyes.phylum,
-    chondrichthyes.class,
-    chondrichthyes.order_,
-    chondrichthyes.family,
-    chondrichthyes.genus,
-    chondrichthyes.category,
-    chondrichthyes.marine,
-    chondrichthyes.terrestial,
-    chondrichthyes.freshwater
-   FROM chondrichthyes
-UNION
- SELECT amphibians.id_no,
-    amphibians.binomial,
-    amphibians.presence,
-    amphibians.origin,
-    amphibians.seasonal,
-    amphibians.legend,
-    amphibians.kingdom,
-    amphibians.phylum,
-    amphibians.class,
-    amphibians.order_,
-    amphibians.family,
-    amphibians.genus,
-    amphibians.category,
-    amphibians.marine,
-    amphibians.terrestial,
-    amphibians.freshwater
-   FROM amphibians
-UNION
- SELECT mammals.id_no,
-    mammals.binomial,
-    mammals.presence,
-    mammals.origin,
-    mammals.seasonal,
-    mammals.legend,
-    mammals.kingdom,
-    mammals.phylum,
-    mammals.class,
-    mammals.order_,
-    mammals.family,
-    mammals.genus,
-    mammals.category,
-    mammals.marine,
-    mammals.terrestial,
-    mammals.freshwater
-   FROM mammals
-  ORDER BY 1;