Merge pull request #100 from Str3am786/soca_3.10

Soca 3.10

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.10
 ENV IN_DOCKER yes
 WORKDIR /soca
 RUN mkdir src

container_run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -x
 
-soca fetch -i Str3am786 --user -o oeg-upm_repos.csv --not_archived  #modify -i ______ to organisation, add --user if its a user
+soca fetch -i oeg-upm --org -o oeg-upm_repos.csv --not_archived  #modify -i ______ to organisation, add --user if its a user
 soca extract -i oeg-upm_repos.csv -o oeg-upm_metadata
 soca portal -i oeg-upm_metadata
 soca summary -i portal/cards_data.json -U

installer.sh

@@ -15,7 +15,7 @@ org=""
 #database Token
 databaseToken=""
 #######################
-echo -e "${token}\n\n\n\n" | somef configure
+echo -e "${token}\n\n\n\n\n" | somef configure
 #
 if [ -z "$databaseToken" ]; then
   databaseToken="\n"

src/soca/assets/about.html

@@ -59,11 +59,13 @@ <h3 style="margin-bottom: 1rem;">Info</h3>
 		<h3 style="margin-bottom: 1rem;">Team</h3>
 		<p style="text-align: justify;">
 			<ul>
-				<li><strong>Daniel Rodríguez Mariblanca</strong>: developer and designer of the portal and soca.
-					<i>Universidad Politécnica de Madrid, ETSIINF</i> (Computer Engineering, Business Administration and Management).</li>
-				<li><strong>Daniel Garijo</strong>: mentor. Distinguished researcher at <i>Universidad Politécnica de Madrid, ETSIINF</i></li>
-				<li><strong>Esteban Gonzalez</strong>: mentor. PhD student at <i>Universidad Politécnica de Madrid, ETSIINF</i></li>
-				<li><strong>Oscar Corcho</strong>: mentor. Full professor at <i>Universidad Politécnica de Madrid, ETSIINF</i></li>
+				<li><strong>Daniel Rodríguez Mariblanca</strong>: Developer and designer of the portal and soca.
+					<i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i> (Computer Engineering, Business Administration and Management).</li>
+				<li><strong>Miguel Arroyo M&aacute;rquez</strong> Developer and Designer for SOCA and SOCA-Dash. 
+					<i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
+				<li><strong>Daniel Garijo</strong>: mentor. Distinguished researcher at <i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
+				<li><strong>Esteban Gonz&aacute;lez</strong>: mentor. PhD student at <i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
+				<li><strong>Oscar Corcho</strong>: mentor. Full professor at <i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
 		</p>
     </article>
 	<article class="soca-card" style="min-height: 26rem; display: inline; min-width: 56.5rem;">

src/soca/assets/template.html

@@ -16,6 +16,7 @@
 		<h2 id="nav-title">Software Catalog</h2>
 		<input id="searchBar" placeholder="Search for repositories..." title="Type in a repository" type="text"/>
 		<div data-toggle="tooltip" data-placement="right" title="About page"><a href="about.html"><img src="img/help.png" class="grey-color-svg" style="height: 2rem; margin-left: 1rem;"></a></div>
+		<div data-toggle="tooltip" data-placement="right" title="SOCA Dashboard Analytics"><a href="https://dashboards-software.oeg.fi.upm.es/d/FX3TNka4k/prototype?orgId=1&from=now-7d&to=now"><img src="img/statIcon.svg" class="grey-color-svg" style="height: 2rem; margin-left: 1rem;"></a></div>
 	</div>
 
 	<div class="filters-wrapper">

src/soca/commands/create_summary.py

@@ -6,6 +6,11 @@
 from .upload_summary import upload_summary
 
 output = {}
+def safe_dic(dic, key):
+    try:
+        return dic[key]
+    except:
+        return None
 def __json_serial(obj):
     """JSON serializer for objects not serializable by default json code
     Returns
@@ -21,17 +26,22 @@ def reset_dict():
     output['has_documentation'] = 0
     output['identifiers'] = {'num_doi': 0, 'num_pid': 0, 'num_without_identifier': 0}
     output['readme'] = {'Level 0': 0, 'Level 1': 0, 'Level 2': 0, 'Level 3': 0}
+    output['num_no_readme'] = 0
+    output['num_with_readme'] = 0
+    output['num_with_description'] = 0
+    output['num_with_acknowledgement'] = 0
+    output['num_with_installation'] = 0
+    output['num_with_requirement'] = 0
     output['licenses'] = {'APACHE': 0, 'MIT': 0, 'GPL': 0, 'OTHER': 0, 'MISSING': 0}
     output['has_citation'] = 0
     output['CFF_file'] = 0
     output['no_citation'] = 0
     output['released'] = {'<2 MONTHS': 0, 'LONGER': 0}
     output['_soca_version'] = soca_ver
     output['_somef_version'] = somef_ver
-    output['_org_name'] = ""
-    #TODO look into portal date/time
     output['_timestamp'] = __json_serial(datetime.now())
     output['num_repos'] = 0
+    output['language_count'] = {}
 
 
 
@@ -100,10 +110,8 @@ def readme_score(json_obj):
             score +=1
         if json_obj['acknowledgement']:
             score +=1
-        #TODO
         if json_obj['requirement']:
             score +=1
-        #TODO
         if json_obj['description'] is not None:
             score +=1
 
@@ -114,6 +122,27 @@ def readme_score(json_obj):
     if score <= 2:
         return "Level 1"
 
+def __readme_analysis(json_obj):
+    """Function that checks if readme exists and counts the number of practices
+        Good practices taken into consideration: installation, acknowledgement, requirement, description
+        Returns
+        ---------
+        Nothing
+        """
+    if not json_obj['readmeUrl']:
+        output['num_no_readme'] += 1
+    else:
+        output['num_with_readme'] +=1
+        if json_obj['installation']:
+            output['num_with_installation'] +=1
+        if json_obj['acknowledgement']:
+            output['num_with_acknowledgment'] +=1
+        if json_obj['requirement']:
+            output['num_with_requirement'] +=1
+        if json_obj['description']:
+            output['num_with_description'] +=1
+
+
 def __findCitation(json_obj):
     """Function that counts the number of repositories per citation
     Returns
@@ -127,6 +156,21 @@ def __findCitation(json_obj):
     else:
         output['no_citation'] += 1
 
+def __languages(json_obj):
+    try:
+        for language in json_obj['languages']:
+            if language in output['language_count']:
+                output['language_count'][language] += 1
+            else:
+                output['language_count'][language] = 1
+    except:
+        if 'ERROR' in output['language_count']:
+            output['language_count']['ERROR'].append(json_obj['id'])
+        else:
+            output['language_count']['ERROR'] = []
+            output['language_count']['ERROR'].append(json_obj['id'])
+
+
 
 #function that opens array of jsons given the organisation
 def __open_Json(directory):
@@ -160,13 +204,14 @@ def create_summary(directory_org_data,outFile, want2Upload):
             # finds identifiers
             __findId(item)
             # gives readme evaluation
-            output['readme'][readme_score(item)] = output['readme'][readme_score(item)] + 1
+            __readme_analysis(item)
             # release time
             output['released'][__last_update(item)] = output['released'][__last_update(item)] + 1
             # adds org_name
             output['_org_name'] = item['owner']
             output['num_repos'] += 1
-
+            #licenses
+            __languages(item)
 
         # saves dictionary to json file
         with open(outFile + "/" + item['owner'] + "_summary.json", 'w+') as out_file:

src/soca/commands/portal/metadata.py

@@ -10,6 +10,7 @@
 from pygments.formatters import HtmlFormatter
 import mistune
 import os
+#from cffconvert.cli import cli as cff2bibcli
 
 class metadata(object):
 
@@ -36,8 +37,7 @@ def html_repo_type(self):
             #ontologies = safe_dic(safe_dic(self.md,'ontologies'),'excerpt')
             ontologies = safe_dic(self.md,'ontologies')
             if ontologies:
-                onto_list = '\n'.join(list(dict.fromkeys([ f'* <{safe_dic(x,"uri")}>' for x in ontologies if 'http' in safe_dic(x,"uri")])))
-                onto_list = '\n'.join(list(dict.fromkeys([ f'* <{safe_dic(x,"uri")}>' for x in ontologies if 'http' in safe_dic(x,"uri")])))
+                onto_list = '\n'.join(list(dict.fromkeys([ f'* <{safe_dic(safe_dic(x,"result"),"value")}>' for x in ontologies if 'http' in safe_dic(safe_dic(x,"result"),"value")])))
                 #onto_list = '\n'.join([ f'* <{safe_dic(x,"file_url")}>' for x in ontologies])
             return self.icon_wrapper(
                 icon_html = f'<img src="{self.base}repo_icons/ontology.png" {self.add_tooltip("left","Ontology")} alt="repo-type" class="repo-type" style="height: 1.3rem;">',
@@ -178,17 +178,26 @@ def html_repo_icons(self):
         citations = self.citations()
         if citations:
             formatter = HtmlFormatter(linenos=False, full=True, style='friendly')
-            for citation in citations:
-                html += self.icon_wrapper(
-                    icon_html = f"""<img src="{self.base}repo_icons/citation.png" 
-                                class="repo-icon" 
-                                {self.add_tooltip('bottom',f"Citation")}>""",
-
-                    modal_html = self.modal(
-                        title = 'Citation',
-                        body = f'<div style="font-family: monospace;">{highlight(citation, ScdocLexer(), formatter)}</div>',
-                        markdown_translation=False,
-                        extra_html=
+            #TODO once fixed turn to if, elif, else  so that it prioritises CFF (converted to bibtex format)
+            if 'cff' in citations:
+                #TODO
+                pass
+            if 'bibtex' in citations:
+                citation = citations['bibtex']
+            else:
+                try:
+                    citation = citations['citation'][0]
+                except Exception as e:
+                    print(str(e))
+            html += self.icon_wrapper(
+                icon_html = f"""<img src="{self.base}repo_icons/citation.png" 
+                            class="repo-icon" 
+                            {self.add_tooltip('bottom',f"Citation")}>""",
+                modal_html = self.modal(
+                    title = 'Citation',
+                    body = f'<div style="font-family: monospace;">{highlight(citation, ScdocLexer(), formatter)}</div>',
+                    markdown_translation=False,
+                    extra_html=
                         f"""
                         <button 
                             class="copy-citation-btn" 
@@ -269,7 +278,14 @@ def html_repo_icons(self):
         hasDocumentation = self.hasDocumentation()
         if hasDocumentation:
             if len(hasDocumentation) > 1:
-                mk_list = "\n".join([f'* <{d}>' if ('http' in d and not ' ' in d) else f'* {d}' for d in hasDocumentation])
+                #mk_list = "\n".join([f'* <{d}>' if ('http' in d and not ' ' in d) else f'* {d}' for d in hasDocumentation])
+                mk_list = "\n".join([
+                    f'* <{safe_dic(safe_dic(d, "result"), "value")}>' if (
+                                'http' in safe_dic(safe_dic(d, "result"), "value") and ' ' not in safe_dic(
+                            safe_dic(d, "result"), "value"))
+                    else f'* {safe_dic(safe_dic(d, "result"), "value")}' for d in hasDocumentation
+                ])
+
                 html += self.icon_wrapper(
                     icon_html = f"""<img src="{self.base}repo_icons/documentation.png" 
                             class="repo-icon" 
@@ -537,7 +553,7 @@ def releases(self):
         return safe_dic(self.md,'releases')
 
     def url_releases(self):
-        return safe_dic(self.md,'download_url')
+        return safe_dic(safe_dic(safe_list(safe_dic(self.md,'download_url'),0),'result'),'value')
 
     def url_stars(self):
         return self.repo_url()+'/stargazers'
@@ -546,8 +562,6 @@ def owner(self):
         return safe_dic(safe_dic(safe_list(safe_dic(self.md,'owner'),0),'result'),'value')
 
 
-
-    #TODO awaiting SOMEF update
     #IMPORTANT !!!!! ASSUMES only 1 CFF per repo
     #USE SOMEF as example it lists SOMEF CFF then WIDOCO then SOMEF then CAPTUM
     def citations(self):
@@ -556,10 +570,11 @@ def citations(self):
         if not all_citations:
             return None
 
-        citations = {'bibtex': [], 'citation': []}
+        citations = { 'citation': []}
 
         for c in all_citations:
             try:
+                type = ""
                 type = c['result']['format']
             except:
                 try:
@@ -572,11 +587,10 @@ def citations(self):
                     citations['cff'] = c['result']['value']
                 case 'bibtex':
                     citations['bibtex'] = c['result']['value']
-
                 case _:
-                    next()
+                    continue
         return citations if len(citations) > 0 else None
-    #TODO ask dani about this paper function. \
+
     # Originally citations Took the ver8 somef "regular expression" output and would create a list of excerpts
 
     def paper(self):

src/soca/commands/single_card.py

@@ -1,5 +1,5 @@
 from soca.commands.portal import card
-from somef.cli import cli_get_data
+from somef.somef_cli import cli_get_data
 from soca import HiddenPrints
 import traceback
 import os