Added additional checks before adding items

crawling/crawler.go

@@ -52,7 +52,7 @@ func (c Crawler) CrawlOnce() (string, error) {
 			fmt.Println(err)
 		}
 	} else {
-		fmt.Println("saving uninitialized...")
+		fmt.Println("saving empty...")
 
 		if err := c.DB.SaveNodeUninitialized(*nextNode); err != nil {
 			fmt.Println(err)

crawling/interfaces.go

@@ -135,4 +135,5 @@ type RPCInterface interface {
 	GetConnections(string) ([]string, error)
 	GetItems(id string) ([]Item, error)
 	GetProfile(id string) (*ProfileResponse, error)
+	GetUserAgent(id string) (string, error)
 }

main.go

@@ -61,6 +61,13 @@ func main() {
 
 			fmt.Printf("Crawling %s\n", nodeID)
 
+			_, err = c.RPCInterface.GetUserAgent(nodeID)
+			if err != nil {
+				done <- true
+				fmt.Printf("Could not access node %s. ignoring\n", nodeID)
+				return
+			}
+
 			items, err := c.RPCInterface.GetItems(nodeID)
 			if err != nil {
 				done <- true

rpc/rpc.go

@@ -132,3 +132,29 @@ func (r OpenBazaarRPC) GetProfile(id string) (*crawling.ProfileResponse, error)
 	}
 	return &response, nil
 }
+
+// GetUserAgent gets the user agent of a specific node
+func (r OpenBazaarRPC) GetUserAgent(id string) (string, error) {
+	req, err := http.NewRequest("GET", "http://"+path.Join(r.URL, "ipns", id, "user_agent"), nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	responseRaw := new(bytes.Buffer)
+	responseRaw.ReadFrom(resp.Body)
+
+	response := responseRaw.String()
+
+	if err := json.Unmarshal(responseRaw.Bytes(), &response); err != nil {
+		fmt.Println(err)
+		var possibleError ErrorResponse
+		if err := json.Unmarshal(responseRaw.Bytes(), &possibleError); err == nil {
+			return "", nil // fail silently
+		}
+		return "", err
+	}
+	return response, nil
+}