mc ls not working with GCS (google cloud storage) with version RELEASE.2020-06-20T00-18-43Z

[manigandham]

This happened before with issue #3073

Happening again with the latest version: RELEASE.2020-06-20T00-18-43Z

root@TITAN:/# mc ls gs/dbfiles-1/
mc: <ERROR> Unable to list folder. A header or query you provided requested a function that is not implemented.

root@TITAN:/# mc ls --json --debug gs/dbfiles-1

mc: <DEBUG> GET /dbfiles-1/?location= HTTP/1.1
Host: storage.googleapis.com
User-Agent: MinIO (linux; amd64) minio-go/v6.0.58 mc/2020-06-20T00:18:43Z
Authorization: AWS4-HMAC-SHA256 Credential=**REDACTED**/20200624/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=**REDACTED**
X-Amz-Content-Sha256: UNSIGNED-PAYLOAD
X-Amz-Date: 20200624T050435Z
Accept-Encoding: gzip

mc: <DEBUG> HTTP/1.1 200 OK
Content-Length: 81
Alt-Svc: h3-27=":443"; ma=2592000,h3-25=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Cache-Control: private, max-age=0
Content-Type: application/xml; charset=UTF-8
Date: Wed, 24 Jun 2020 05:04:39 GMT
Expires: Wed, 24 Jun 2020 05:04:39 GMT
Server: UploadServer
X-Goog-Metageneration: 11
X-Guploader-Uploadid: AAANsUkErGgemMCvsjYFOjWGv4Kx4UXv76fIGE8XJx52muOoHGVwx_iDJmDgn7k9arB-405EH28Nub53d4LeniJdPrM

mc: <DEBUG> Response Time:  285.0191ms

mc: <DEBUG> HEAD / HTTP/1.1
Host: dbfiles-1.storage.googleapis.com
User-Agent: MinIO (linux; amd64) minio-go/v6.0.58 mc/2020-06-20T00:18:43Z
Authorization: AWS4-HMAC-SHA256 Credential=**REDACTED**/20200624/US/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=**REDACTED**
X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
X-Amz-Date: 20200624T050435Z

mc: <DEBUG> HTTP/1.1 200 OK
Content-Length: 1198
Alt-Svc: h3-28=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-25=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q049=":443"; ma=2592000,h3-Q048=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Cache-Control: private, max-age=0
Content-Type: application/xml; charset=UTF-8
Date: Wed, 24 Jun 2020 05:04:40 GMT
Expires: Wed, 24 Jun 2020 05:04:40 GMT
Server: UploadServer
X-Goog-Metageneration: 11
X-Guploader-Uploadid: AAANsUk618pqVOgVQh303GMeumpH1xqKRHi-8dwtNlIlzOahqa9zNjr2C3pkhDizghvOHwe1pXg8-vfxs7kdGUAU3Dg

mc: <DEBUG> Response Time:  291.0367ms

mc: <DEBUG> HEAD / HTTP/1.1
Host: dbfiles-1.storage.googleapis.com
User-Agent: MinIO (linux; amd64) minio-go/v6.0.58 mc/2020-06-20T00:18:43Z
Authorization: AWS4-HMAC-SHA256 Credential=**REDACTED**/20200624/US/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=**REDACTED**
X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
X-Amz-Date: 20200624T050435Z

mc: <DEBUG> HTTP/1.1 200 OK
Content-Length: 1198
Alt-Svc: h3-28=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-25=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q049=":443"; ma=2592000,h3-Q048=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Cache-Control: private, max-age=0
Content-Type: application/xml; charset=UTF-8
Date: Wed, 24 Jun 2020 05:04:40 GMT
Expires: Wed, 24 Jun 2020 05:04:40 GMT
Server: UploadServer
X-Goog-Metageneration: 11
X-Guploader-Uploadid: AAANsUknx4nOW6GWxI0WCKpy_xXez05WSf42mSrgQ6961lHrg4CLcSB0T41oVh3qNaH_Qux-3NYnpdR_5AusP2guLbM

mc: <DEBUG> Response Time:  243.2812ms

mc: <DEBUG> GET /?delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&prefix= HTTP/1.1
Host: dbfiles-1.storage.googleapis.com
User-Agent: MinIO (linux; amd64) minio-go/v6.0.58 mc/2020-06-20T00:18:43Z
Authorization: AWS4-HMAC-SHA256 Credential=**REDACTED**/20200624/US/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=**REDACTED**
X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
X-Amz-Date: 20200624T050435Z
Accept-Encoding: gzip

mc: <DEBUG> HTTP/1.1 400 Bad Request
Content-Length: 239
Alt-Svc: h3-28=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-25=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q049=":443"; ma=2592000,h3-Q048=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Cache-Control: private, max-age=0
Content-Type: application/xml; charset=UTF-8
Date: Wed, 24 Jun 2020 05:04:40 GMT
Expires: Wed, 24 Jun 2020 05:04:40 GMT
Server: UploadServer
X-Guploader-Uploadid: AAANsUkOGhm2UgFvmMBVIW3T3e4O1uBXH5HUgQ288D-E3TJTEoObecm83qIQJ_wiW8UTqeEs3MA9wmX5UCDqm9ORruU

<?xml version='1.0' encoding='UTF-8'?><Error><Code>NotImplemented</Code><Message>A header or query you provided requested a function that is not implemented.</Message><Details>GET ?list-type is not implemented for buckets</Details></Error>mc: <DEBUG> Response Time:  240.1246ms

{
 "status": "error",
 "error": {
  "message": "Unable to list folder.",
  "cause": {
   "message": "A header or query you provided requested a function that is not implemented.",
   "error": {
    "Code": "NotImplemented",
    "Message": "A header or query you provided requested a function that is not implemented.",
    "BucketName": "",
    "Key": "",
    "RequestID": "",
    "HostID": "",
    "Region": "",
    "Server": "UploadServer"
   }
  },
  "type": "error",
  "trace": [
   {
    "line": 2005,
    "file": "client-s3.go",
    "func": "cmd.(*S3Client).listInRoutine"
   },
   {
    "line": 135,
    "file": "ls.go",
    "func": "cmd.doList",
    "env": {
     "Tags": [
      "https://storage.googleapis.com/dbfiles-1/"
     ]
    }
   }
  ],
  "sysinfo": {
   "host.arch": "amd64",
   "host.cpus": "24",
   "host.lang": "go1.13.12",
   "host.name": "TITAN",
   "host.os": "linux",
   "mem.heap.total": "66 MB",
   "mem.heap.used": "2.6 MB",
   "mem.total": "73 MB",
   "mem.used": "2.6 MB"
  }
 }
}

[vadmeste]

@manigandham you have this error because GCS does not support S3 listing v2 but I suppose there is a workaround.

What endpoint did you enter when configuring your mc alias ?

[manigandham]

Endpoint is https://storage.googleapis.com

[manigandham]

Same problem was fixed before in issue #3073

[harshavardhana]

Use older release for now, it is unfortunate that Google lacks latest S3 features and we have to keep these hacks on our end.

Please push for latest S3 APIs on GCS so we can avoid these hacks in future.

[manigandham]

Thanks, that's what we're doing but why close this issue?

Is the intention not to support GCS until they have an upgraded S3 API? Would it better to just have GCS native API support (the way Azure storage does)?

[harshavardhana]

Thanks, that's what we're doing but why close this issue?

Is the intention not to support GCS until they have an upgraded S3 API? Would it better to just have GCS native API support (the way Azure storage does)?

Because we just fixed the issue in mc and will make release @manigandham