Merge branch 'release/0.4.11'

common_osint_model/censys/v1.py

@@ -14,7 +14,7 @@ def from_censys_ipv4(raw: dict) -> dict:
     :param raw: Censys IPv4 dict
     :return: Common format dict
     """
-    warnings.warn("This function was deprecated in v0.4.0.", DeprecationWarning)
+    warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning)
     flattened = False
     for k in raw.keys():
         if "." in k:
@@ -44,7 +44,7 @@ def from_censys_ipv4_flattened(raw: dict) -> dict:
     :param raw: Censys IPv4 dict
     :return: Common format dict, flattened
     """
-    warnings.warn("This function was deprecated in v0.4.0.", DeprecationWarning)
+    warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning)
     return flatten(from_censys_ipv4(raw))
 
 

common_osint_model/censys/v2.py

@@ -13,7 +13,7 @@ def from_censys(raw: Dict) -> Dict:
 
     :param raw: Censys Search 2.0 dictionary
     """
-    warnings.warn("This function was deprecated in v0.4.0.", DeprecationWarning)
+    warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning)
     common = {}
     common.update(
         censys_meta_extraction(raw)
@@ -25,7 +25,7 @@ def from_censys(raw: Dict) -> Dict:
 
 
 def from_censys_flattened(raw: Dict) -> Dict:
-    warnings.warn("This function was deprecated in v0.4.0.", DeprecationWarning)
+    warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning)
     return flatten(from_censys(raw))
 
 

common_osint_model/models/domain.py

@@ -11,14 +11,16 @@ class Entity(BaseModel):
     organization: Optional[str]
     street: Optional[str]
     city: Optional[str]
+    state: Optional[str]
     postal_code: Optional[str]
     country: Optional[str]
     phone: Optional[str]
     timestamp: datetime = datetime.utcnow()
 
 
 class Domain(BaseModel):
-    """Represents a domain pointing to a specific host."""
+    """Represents a domain pointing to a specific host. Also, this object might be used to represent found via other
+    sources, therefore a 'query' field might contain the query used to find it"""
     domain: str
     first_seen: datetime = datetime.utcnow()
     last_seen: datetime = datetime.utcnow()
@@ -28,3 +30,4 @@ class Domain(BaseModel):
     nameserver: Optional[List[str]]
     registrar: Optional[str]
     registrant: Optional[Entity]
+    query: Optional[str]

common_osint_model/models/ssh.py

@@ -175,6 +175,7 @@ class SSHComponent(BaseModel, ShodanDataHandler, CensysDataHandler, BinaryEdgeDa
     """Represents the SSH component of services."""
     algorithms: Optional[SSHComponentAlgorithms]
     key: Optional[SSHComponentKey]
+    hassh: Optional[str]
 
     @classmethod
     def from_shodan(cls, d: Dict):
@@ -183,17 +184,22 @@ def from_shodan(cls, d: Dict):
             raise TypeError(f"Method SSHComponent.from_shodan expects parameter d to be a dictionary, "
                             f"but it was {type(d)}.")
 
+        hassh = d.get("ssh", {}).get("hashh", None)
+
         return SSHComponent(
             algorithms=SSHComponentAlgorithms.from_shodan(d),
-            key=SSHComponentKey.from_shodan(d)
+            key=SSHComponentKey.from_shodan(d),
+            hassh=hassh
         )
 
     @classmethod
     def from_censys(cls, d: Dict):
         """Creates an instance of this class based on Censys data given as dictionary."""
+        hassh = d.get("ssh", {}).get("hassh_fingerprint", None)
         return SSHComponent(
             algorithms=SSHComponentAlgorithms.from_censys(d),
-            key=SSHComponentKey.from_censys(d)
+            key=SSHComponentKey.from_censys(d),
+            hassh=hassh
         )
 
     @classmethod
@@ -202,14 +208,16 @@ def from_binaryedge(cls, d: Dict) -> Union["SSHComponent", None]:
         try:
             cyphers = d["result"]["data"]["cyphers"]
             algorithms = d["result"]["data"]["algorithms"]
+            hassh = d["result"]["data"]["hassh"]["hassh"]
             cypher = None
             for c in cyphers:
                 if c["cypher"] == "ssh-dss":
                     continue
                 cypher = c
             return SSHComponent(
                 algorithms=SSHComponentAlgorithms.from_binaryedge(algorithms),
-                key=SSHComponentKey.from_binaryedge(cypher)
+                key=SSHComponentKey.from_binaryedge(cypher),
+                hassh=hassh
             )
         except KeyError as ke:
             cls.warning(f"BinaryEdge data is missing key: {ke}")

common_osint_model/shodan.py

@@ -17,7 +17,7 @@ def from_shodan(raw: Union[list, dict]) -> dict:
     :param raw: Shodan list or dictionary from host queries
     :return: Generic host describing dictionary
     """
-    warnings.warn("This function was deprecated in v0.4.0.", DeprecationWarning)
+    warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning)
     g = {}
     services = []
     if isinstance(raw, dict):
@@ -54,7 +54,7 @@ def from_shodan_flattened(raw: Union[list, dict]) -> dict:
     :param raw: Shodan list or dictionary from host queries
     :return: Generic host describing dictionary, flattened
     """
-    warnings.warn("This function was deprecated in v0.4.0.", DeprecationWarning)
+    warnings.warn("This function was deprecated in v0.4.0. and will be removed in v0.5.0", DeprecationWarning)
     return flatten(from_shodan(raw))
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "common-osint-model"
-version = "0.4.10"
+version = "0.4.11"
 description = "Converting data from services like BinaryEdge, Censys and Shodan to a common data model."
 authors = ["3c7 <3c7@posteo.de>"]
 license = "MIT"