-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add config package * Refactor scraper main.go file * Remove unused init function * fix error message typo * update comments * Refactor file type handling in config and main.go * move ip check function to utils package * Remove unused variables and comments * Refactor JSON file writing and review sorting * comments * comments * formatting
- Loading branch information
Showing
6 changed files
with
143 additions
and
105 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,5 @@ | ||
| module github.com/algo7/TripAdvisor-Review-Scraper/scraper | ||
|
|
||
| go 1.21 | ||
| go 1.21.1 | ||
|
|
||
| toolchain go1.21.4 |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| package config | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "strings" | ||
| ) | ||
|
|
||
| // Config is a struct that represents the configuration for the scraper | ||
| type Config struct { | ||
| LocationURL string | ||
| Languages []string | ||
| FileType string | ||
| ProxyHost string | ||
| } | ||
|
|
||
| // NewConfig is a function that returns a new Config struct | ||
| // Returns an error if the LOCATION_URL is not set | ||
| func NewConfig() (*Config, error) { | ||
| // Default languages | ||
| defaultLanguages := []string{"en"} | ||
|
|
||
| // Get location URL | ||
| locationURL := os.Getenv("LOCATION_URL") | ||
| if locationURL == "" { | ||
| return nil, fmt.Errorf("LOCATION_URL not set") | ||
| } | ||
|
|
||
| // Get languages | ||
| languages := defaultLanguages | ||
| if envLang := os.Getenv("LANGUAGES"); envLang != "" { | ||
| languages = strings.Split(envLang, "|") | ||
| } | ||
|
|
||
| // Get file type | ||
| fileType := strings.ToLower(os.Getenv("FILETYPE")) | ||
| if fileType == "" { | ||
| fileType = "csv" | ||
| } | ||
|
|
||
| if fileType != "csv" && fileType != "json" { | ||
| return nil, fmt.Errorf("invalid file type. Use csv or json") | ||
| } | ||
|
|
||
| // Get proxy host | ||
| proxyHost := os.Getenv("PROXY_HOST") | ||
|
|
||
| return &Config{ | ||
| LocationURL: locationURL, | ||
| Languages: languages, | ||
| FileType: fileType, | ||
| ProxyHost: proxyHost, | ||
| }, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package utils | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| ) | ||
|
|
||
| // CheckIP takes in a http client and calls ipinfo.io/ip to check the current IP address | ||
| func CheckIP(client *http.Client) (ip string, err error) { | ||
|
|
||
| // Make the request to ipinfo.io/ip | ||
| resp, err := client.Get("https://ipinfo.io/ip") | ||
| if err != nil { | ||
| return "", fmt.Errorf("error getting IP address: %w", err) | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| // Check the response status code | ||
| if resp.StatusCode != http.StatusOK { | ||
| return "", fmt.Errorf("error response status code: %d", resp.StatusCode) | ||
| } | ||
|
|
||
| // Read the response body | ||
| responseBody, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return "", fmt.Errorf("error reading response body: %w", err) | ||
| } | ||
|
|
||
| return string(responseBody), nil | ||
| } |