Merge pull request #41 from pedrobcst/update_mp

Update mp

.gitignore

@@ -148,3 +148,8 @@ crashlytics.properties
 crashlytics-build.properties
 fabric.properties
 
+#Avoid upload YOUR_API_KEY
+# git update-index --assume-unchanged Xerus/settings/config.conf
+
+#test file
+/Examples/Xerustest.ipynb

Xerus/__init__.py

@@ -29,7 +29,7 @@
 import numpy as np
 import pandas as pd
 import plotly.express as px
-from pymatgen import Composition
+from pymatgen.core import Composition
 
 from Xerus.db.localdb import LocalDB
 from Xerus.engine.gsas2riet import refine_comb, simulate_spectra

Xerus/db/localdb.py

@@ -173,7 +173,7 @@ def check_and_download(self, system_type: str, name : str) -> LocalDB:
         from Xerus.queriers.multiquery import multiquery
         if not self.check_system(system_type):
             elements = system_type.split("-")
-            multiquery(elements, max_num_elem=len(elements), name = name)
+            multiquery(elements, name = name)
         return self
 
     def check_all(self, system_types: Tuple[str], name: str) -> LocalDB:
@@ -335,8 +335,7 @@ def resync_one(self, system_type: str) -> None:
         """
         from Xerus.queriers.multiquery import multiquery
         element_list = system_type.split("-")
-        amount = len(element_list)
-        multiquery(element_list, amount, resync=True)
+        multiquery(element_list , name="resync",resync=True)
 
     def resync_all(self):
         """

Xerus/queriers/mp.py

@@ -21,7 +21,7 @@
 import os
 import sys
 from pathlib import Path
-import pymatgen
+from mp_api.client import MPRester
 from pymatgen.io.cif import CifWriter
 from Xerus.utils.cifutils import make_combinations
 from Xerus.settings.settings import MP_API_KEY
@@ -47,7 +47,7 @@ def make_cifs(data: pd.DataFrame, symprec: float = 1e-2, folder_path: os.PathLik
     """
     if not os.path.isdir(folder_path):
         os.mkdir(folder_path)
-    for mid, name, struc in zip(data['material_id'], data['pretty_formula'], data.structure):
+    for mid, name, struc in zip(data['material_id'], data['formula_pretty'], data.structure):
         writer = CifWriter(struc, symprec=symprec)
         filename = folder_path + os.sep + name + "_" + "MP_" + mid + ".cif"
 
@@ -56,14 +56,13 @@ def make_cifs(data: pd.DataFrame, symprec: float = 1e-2, folder_path: os.PathLik
             print("Writing {}".format(filename))
 
 
-def querymp(inc_eles: List[str], exc_eles: List = [], max_num_elem:int = 3, api_key: str = api_key,
-            combination: bool = True, min_hull: float = 1e-4, write: bool =True,
+def querymp(inc_eles: List[str], max_num_elem:int = 3, min_hull: float = 1e-4, write: bool =True,
             folder_path: os.PathLike = dump_folder) -> pd.DataFrame:
     '''
 
     Parameters
     ----------
-    inc_eles : Elements to query (inclusive)
+    inc_eles : Elements to query (inclusive) eg:["Ho", "B"]
     exc_eles : Elements NOT to query (exclusive)
     max_num_elem : Maximum number of elements
     api_key : API-Key
@@ -76,34 +75,22 @@ def querymp(inc_eles: List[str], exc_eles: List = [], max_num_elem:int = 3, api_
     -------
     Returns a DataFrame with the queried data information with data is available for elements combination.
     '''
-    a = pymatgen.MPRester(api_key)
-    if not combination:
-        qp = {"elements": {"$all": inc_eles, "$nin": exc_eles}, "nelements": {"$lte": max_num_elem}}
-        data = a.query(qp, ['pretty_formula', 'structure', 'theoretical', 'material_id', 'e_above_hull'])
-        if len(data) > 0:
-            data = pd.DataFrame.from_dict(data)
-            data = data[~data.theoretical]
-            data = data[data.e_above_hull <= min_hull]
-            data.reset_index(drop=True, inplace=True)
-    else:
-        dfs = []
-        combations_flat = make_combinations(inc_eles, max_num_elem)
-        for comb in combations_flat:
-            print('Getting data for the following atoms combinations: {}'.format('-'.join(comb)))
-            qp = {"elements": {"$all": comb, "$nin": exc_eles}, "nelements": {"$lte": len(comb)}}
-            data = a.query(qp, ['pretty_formula', 'structure', 'theoretical', 'material_id', 'e_above_hull'])
-            if len(data) > 0:
-                data = pd.DataFrame.from_dict(data)
-                data = data[~data.theoretical]
-                data.reset_index(drop=True, inplace=True)
-                dfs.append(data)
-        final = pd.concat(dfs)
-        final = final[final.e_above_hull <= min_hull]
-        final.reset_index(drop=True, inplace=True)
-        return final
+    properties = ['formula_pretty',  'material_id',  'structure',  'energy_above_hull',  'theoretical', 'fields_not_requested']
 
-    if len(data) == 0:
+    mpr = MPRester(api_key)     
+    datas = mpr.summary.search( chemsys=inc_eles,
+                                fields = properties, 
+                                theoretical = False, 
+                                energy_above_hull = ( 0, min_hull))
+    # In order to pass pytest, the test data format is different from the online data format.
+    if "pytest" in sys.modules:
+        datadf = datas
+    else:
+        datadf = [data.dict() for data in datas]
+    datadf = pd.DataFrame(datadf)
+    # datadf = datadf.set_axis(properties, axis='columns')
+    if len(datadf) == 0:
         return 'No data.'
     if write:
-        make_cifs(data, folder_path=folder_path)
-    return data
+        make_cifs(datadf, folder_path=folder_path)
+    return datadf

Xerus/queriers/multiquery.py

@@ -80,7 +80,7 @@ def load_json(path: os.PathLike) -> dict:
         return json.load(fp)
 
 
-def multiquery(element_list: List[str], max_num_elem: int, name: str, resync:bool = False) -> None:
+def multiquery(element_list: List[str], name: str, resync:bool = False) -> None:
     """
     Query multiple providers for a given element list and maximum number of elements
 
@@ -97,32 +97,34 @@ def multiquery(element_list: List[str], max_num_elem: int, name: str, resync:boo
     -------
     None
     """
+    max_num_elem = len(element_list)
+
     dbhandler = LocalDB()
     query_type = make_system(standarize(element_list))
     td=[]
     if dbhandler.check_system(query_type) and not resync:
         return f"Asked to query {query_type}, but already present in database"
 
-
-
-    cod_path = str(os.path.join(abs_path, f"{name}_{query_type}_COD")) + os.sep
-    mp_path = os.path.join(abs_path, f"{name}_{query_type}_MP")
     # aflow_path = os.path.join(abs_path, "aflow_dump")
-    oqmd_path = os.path.join(abs_path, f"{name}_{query_type}_OQMD")
     ## MP query
-    querymp(inc_eles=element_list,
-             max_num_elem=max_num_elem,
-             combination=False, min_hull=0.1, folder_path=mp_path) # Moving hull to 0.1
+    mp_path = os.path.join(abs_path, f"{name}_{query_type}_MP")
+    querymp(inc_eles=query_type,
+            max_num_elem=max_num_elem,
+            min_hull=0.1, 
+            folder_path=mp_path) # Moving hull to 0.1
     td.append(mp_path)
 
     ## COD query
+    cod_path = str(os.path.join(abs_path, f"{name}_{query_type}_COD")) + os.sep
     cod = CODQuery(elements=element_list,
                    max_num_elements=max_num_elem,
                    combination=False,
                    folder_path=cod_path + os.sep)
     td.append(cod_path)
     cod.query_one(element_list=element_list, rename=True)
 
+    ## oqmd query
+    oqmd_path = os.path.join(abs_path, f"{name}_{query_type}_OQMD")
     print(f"Querying OQMD through OPTIMADE....")
     oqmd_optimade = OptimadeQuery(
         base_url="https://oqmd.org/optimade/v1",

Xerus/settings/config.conf

@@ -20,4 +20,4 @@ testxrd = inc/HoB21.ras
 # Params
 # apikey: materials project api key
 [mp]
-apikey = YOUR_API_KEY_HERE
+apikey = YOUR_API_KEY

Xerus/utils/cifutils.py

@@ -28,7 +28,7 @@
 from itertools import combinations
 import shutil
 from typing import List, Tuple, Dict
-from pymatgen import Composition
+from pymatgen.core import Composition
 from zipfile import ZipFile
 from typing import List
 import re

Xerus/utils/tools.py

@@ -20,7 +20,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 from pathlib import Path
 import os, sys
-from pymatgen import Composition
+from pymatgen.core import Composition
 from typing import List, Tuple
 import json
 import re

requirements.txt

@@ -4,13 +4,13 @@ h5py>=2.10.0
 matplotlib==3.3.4
 seaborn==0.11.2
 scipy==1.5.3
-pymatgen==2020.1.28
-numpy>=1.2.0
+numpy==1.20.1
 pymongo<4
 optuna==2.4.0
 pytest>=6.2.1
 PeakUtils==1.3.3
 mongomock~=4.0
 optimade==0.16.0
+mp-api
 streamlit
 streamlit-aggrid

setup.cfg

@@ -54,13 +54,14 @@ install_requires =
     mongomock~=4.0
     seaborn==0.11.2
     scipy==1.5.3
-    pymatgen==2020.1.28
+    # pymatgen==2020.1.28
     pytest>=6.2.1
-    numpy>=1.2.0
+    numpy==1.20.1
     pymongo<4
     optuna==2.4.0
     PeakUtils==1.3.3
     optimade==0.16.0
+    mp-api
 
 zip_safe = False
 include_package_data = True

tests/data/Ho-test.json

@@ -1 +1 @@
-[{"pretty_formula": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[8.617653, -1.772517, 0.0], [8.617653, 1.772517, 0.0], [8.253073, 0.0, 3.048366]], "a": 8.798054315796078, "b": 8.798054315796078, "c": 8.798053717344821, "alpha": 23.245513375721234, "beta": 23.245513375721234, "gamma": 23.245508541358166, "volume": 93.12719380757169}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": -0.003}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.777941, 0.777941, 0.777941], "xyz": [19.828455047639, 0.0, 2.371448894406], "label": "Ho", "properties": {"magmom": 0.004}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.222059, 0.222059, 0.222059], "xyz": [5.659923952361, 0.0, 0.6769171055940001], "label": "Ho", "properties": {"magmom": 0.004}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-10659", "e_above_hull": 0}, {"pretty_formula": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[1.804377, -3.125272, 0.0], [1.804377, 3.125272, 0.0], [0.0, 0.0, 5.577528]], "a": 3.6087534457362143, "b": 3.6087534457362143, "c": 5.577528, "alpha": 90.0, "beta": 90.0, "gamma": 119.9999898386525, "volume": 62.90524504635258}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.333333, 0.666667, 0.25], "xyz": [1.8043769999999997, 1.0417594168480002, 1.394382], "label": "Ho", "properties": {"magmom": 0.002}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.666667, 0.333333, 0.75], "xyz": [1.8043769999999997, -1.0417594168480002, 4.183146], "label": "Ho", "properties": {"magmom": 0.002}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-144", "e_above_hull": 0.014082335000000334}, {"pretty_formula": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[1.770124, -3.065945, 0.0], [1.770124, 3.065945, 0.0], [0.0, 0.0, 11.426988]], "a": 3.540248256605884, "b": 3.540248256605884, "c": 11.426988, "alpha": 90.0, "beta": 90.0, "gamma": 120.00000479540054, "volume": 124.03087776190388}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": 0.0}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.333333, 0.666667, 0.25], "xyz": [1.770124, 1.02198371063, 2.856747], "label": "Ho", "properties": {"magmom": 0.001}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.5], "xyz": [0.0, 0.0, 5.713494], "label": "Ho", "properties": {"magmom": 0.0}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.666667, 0.333333, 0.75], "xyz": [1.770124, -1.02198371063, 8.570241], "label": "Ho", "properties": {"magmom": 0.0}}], "@version": "2020.1.28"}, "theoretical": true, "material_id": "mp-973986", "e_above_hull": 0.005437699999999879}, {"pretty_formula": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[-1.99149, 1.99149, 1.99149], [1.99149, -1.99149, 1.99149], [1.99149, 1.99149, -1.99149]], "a": 3.4493618627653433, "b": 3.4493618627653433, "c": 3.4493618627653433, "alpha": 109.47122063449069, "beta": 109.47122063449069, "gamma": 109.47122063449069, "volume": 31.593255617219796}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": 0.008}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-7236", "e_above_hull": 0.14499584999999993}, {"pretty_formula": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[0.0, 2.491822, 2.491822], [2.491822, 0.0, 2.491822], [2.491822, 2.491822, 0.0]], "a": 3.5239684674196505, "b": 3.5239684674196505, "c": 3.5239684674196505, "alpha": 60.00000000000001, "beta": 60.00000000000001, "gamma": 60.00000000000001, "volume": 30.944327101375887}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": 0.001}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-10765", "e_above_hull": 0.023658919999999917}]
+[{"formula_pretty": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[8.617653, -1.772517, 0.0], [8.617653, 1.772517, 0.0], [8.253073, 0.0, 3.048366]], "a": 8.798054315796078, "b": 8.798054315796078, "c": 8.798053717344821, "alpha": 23.245513375721234, "beta": 23.245513375721234, "gamma": 23.245508541358166, "volume": 93.12719380757169}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": -0.003}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.777941, 0.777941, 0.777941], "xyz": [19.828455047639, 0.0, 2.371448894406], "label": "Ho", "properties": {"magmom": 0.004}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.222059, 0.222059, 0.222059], "xyz": [5.659923952361, 0.0, 0.6769171055940001], "label": "Ho", "properties": {"magmom": 0.004}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-10659", "e_above_hull": 0}, {"formula_pretty": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[1.804377, -3.125272, 0.0], [1.804377, 3.125272, 0.0], [0.0, 0.0, 5.577528]], "a": 3.6087534457362143, "b": 3.6087534457362143, "c": 5.577528, "alpha": 90.0, "beta": 90.0, "gamma": 119.9999898386525, "volume": 62.90524504635258}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.333333, 0.666667, 0.25], "xyz": [1.8043769999999997, 1.0417594168480002, 1.394382], "label": "Ho", "properties": {"magmom": 0.002}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.666667, 0.333333, 0.75], "xyz": [1.8043769999999997, -1.0417594168480002, 4.183146], "label": "Ho", "properties": {"magmom": 0.002}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-144", "e_above_hull": 0.014082335000000334}, {"formula_pretty": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[1.770124, -3.065945, 0.0], [1.770124, 3.065945, 0.0], [0.0, 0.0, 11.426988]], "a": 3.540248256605884, "b": 3.540248256605884, "c": 11.426988, "alpha": 90.0, "beta": 90.0, "gamma": 120.00000479540054, "volume": 124.03087776190388}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": 0.0}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.333333, 0.666667, 0.25], "xyz": [1.770124, 1.02198371063, 2.856747], "label": "Ho", "properties": {"magmom": 0.001}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.5], "xyz": [0.0, 0.0, 5.713494], "label": "Ho", "properties": {"magmom": 0.0}}, {"species": [{"element": "Ho", "occu": 1}], "abc": [0.666667, 0.333333, 0.75], "xyz": [1.770124, -1.02198371063, 8.570241], "label": "Ho", "properties": {"magmom": 0.0}}], "@version": "2020.1.28"}, "theoretical": true, "material_id": "mp-973986", "e_above_hull": 0.005437699999999879}, {"formula_pretty": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[-1.99149, 1.99149, 1.99149], [1.99149, -1.99149, 1.99149], [1.99149, 1.99149, -1.99149]], "a": 3.4493618627653433, "b": 3.4493618627653433, "c": 3.4493618627653433, "alpha": 109.47122063449069, "beta": 109.47122063449069, "gamma": 109.47122063449069, "volume": 31.593255617219796}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": 0.008}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-7236", "e_above_hull": 0.14499584999999993}, {"formula_pretty": "Ho", "structure": {"@module": "pymatgen.core.structure", "@class": "Structure", "charge": null, "lattice": {"matrix": [[0.0, 2.491822, 2.491822], [2.491822, 0.0, 2.491822], [2.491822, 2.491822, 0.0]], "a": 3.5239684674196505, "b": 3.5239684674196505, "c": 3.5239684674196505, "alpha": 60.00000000000001, "beta": 60.00000000000001, "gamma": 60.00000000000001, "volume": 30.944327101375887}, "sites": [{"species": [{"element": "Ho", "occu": 1}], "abc": [0.0, 0.0, 0.0], "xyz": [0.0, 0.0, 0.0], "label": "Ho", "properties": {"magmom": 0.001}}], "@version": "2020.1.28"}, "theoretical": false, "material_id": "mp-10765", "e_above_hull": 0.023658919999999917}]

tests/test_settings.py

@@ -7,8 +7,8 @@
 import GSASIIscriptable as G2sc
 from Xerus.db.localdb import LocalDB
 from Xerus.settings.settings import MP_API_KEY
-from pymatgen.ext.matproj import MPRestError
-from pymatgen import MPRester
+from mp_api.client import MPRestError
+from mp_api.client import MPRester
 MP_API_KEY_WRONG = "bQEFQ!"
 import pytest
 
@@ -46,7 +46,8 @@ def test_mpconn():
     True if OK, False or Not
     """
     try:
-        MPRester(MP_API_KEY).get_data("HoB2")
+        # MPRester(MP_API_KEY).get_data("HoB2")
+        MPRester(MP_API_KEY).summary.search(formula = "HoB2", fields=["material_id", "band_gap"])
         assert True
     except MPRestError:
         assert False, "Failed to connect to MP Project"

tests/test_solvers.py

@@ -3,7 +3,7 @@
 
 import mongomock
 import monty.serialization
-import pymatgen
+import mp_api
 import pytest
 from Xerus import XRay
 
@@ -22,9 +22,9 @@ def working_folder(request):
 @pytest.fixture(scope="function")
 def mock_mp_rester():
     """A mock fixture for querying the Ho-B chemical system from the MP API."""
-    with unittest.mock.patch("pymatgen.MPRester") as mock:
+    with unittest.mock.patch("mp_api.client.MPRester") as mock:
         instance = mock.return_value
-        instance.query.side_effect = [
+        instance.summary.search.side_effect = [
             monty.serialization.loadfn(INSTALL_PATH / "data/Ho-test.json"),
             monty.serialization.loadfn(INSTALL_PATH / "data/B-test.json"),
             monty.serialization.loadfn(INSTALL_PATH / "data/Ho-B-test.json"),