From 81068bc1ce423ab7ffc7db4f3b088bd9bb113bff Mon Sep 17 00:00:00 2001 From: Artur Rodrigues Date: Thu, 5 Jan 2023 15:27:37 +0000 Subject: [PATCH 1/4] Remove unused cloudflare pagination options Signed-off-by: Artur Rodrigues --- main.go | 2 +- pkg/apis/externaldns/types.go | 3 --- pkg/apis/externaldns/types_test.go | 4 ---- provider/cloudflare/cloudflare.go | 16 +++++----------- provider/cloudflare/cloudflare_test.go | 3 --- 5 files changed, 6 insertions(+), 22 deletions(-) diff --git a/main.go b/main.go index df8eabb13a..815afa0b01 100644 --- a/main.go +++ b/main.go @@ -233,7 +233,7 @@ func main() { case "civo": p, err = civo.NewCivoProvider(domainFilter, cfg.DryRun) case "cloudflare": - p, err = cloudflare.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareZonesPerPage, cfg.CloudflareProxied, cfg.DryRun) + p, err = cloudflare.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun) case "rcodezero": p, err = rcode0.NewRcodeZeroProvider(domainFilter, cfg.DryRun, cfg.RcodezeroTXTEncrypt) case "google": diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index 8224c80ee4..3bd9a61d70 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -105,7 +105,6 @@ type Config struct { BluecatDNSDeployType string BluecatSkipTLSVerify bool CloudflareProxied bool - CloudflareZonesPerPage int CoreDNSPrefix string RcodezeroTXTEncrypt bool AkamaiServiceConsumerDomain string @@ -253,7 +252,6 @@ var defaultConfig = &Config{ BluecatConfigFile: "/etc/kubernetes/bluecat.json", BluecatDNSDeployType: "no-deploy", CloudflareProxied: false, - CloudflareZonesPerPage: 50, CoreDNSPrefix: "/skydns/", RcodezeroTXTEncrypt: false, AkamaiServiceConsumerDomain: "", @@ -472,7 +470,6 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("bluecat-dns-deploy-type", "When using the Bluecat provider, specify the type of DNS deployment to initiate after records are updated. Valid options are 'full-deploy' and 'no-deploy'. Deploy will only execute if --bluecat-dns-server-name is set (optional when --provider=bluecat)").Default(defaultConfig.BluecatDNSDeployType).StringVar(&cfg.BluecatDNSDeployType) app.Flag("cloudflare-proxied", "When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled)").BoolVar(&cfg.CloudflareProxied) - app.Flag("cloudflare-zones-per-page", "When using the Cloudflare provider, specify how many zones per page listed, max. possible 50 (default: 50)").Default(strconv.Itoa(defaultConfig.CloudflareZonesPerPage)).IntVar(&cfg.CloudflareZonesPerPage) app.Flag("coredns-prefix", "When using the CoreDNS provider, specify the prefix name").Default(defaultConfig.CoreDNSPrefix).StringVar(&cfg.CoreDNSPrefix) app.Flag("akamai-serviceconsumerdomain", "When using the Akamai provider, specify the base URL (required when --provider=akamai and edgerc-path not specified)").Default(defaultConfig.AkamaiServiceConsumerDomain).StringVar(&cfg.AkamaiServiceConsumerDomain) app.Flag("akamai-client-token", "When using the Akamai provider, specify the client token (required when --provider=akamai and edgerc-path not specified)").Default(defaultConfig.AkamaiClientToken).StringVar(&cfg.AkamaiClientToken) diff --git a/pkg/apis/externaldns/types_test.go b/pkg/apis/externaldns/types_test.go index 6cd42b5b4f..24a699b38d 100644 --- a/pkg/apis/externaldns/types_test.go +++ b/pkg/apis/externaldns/types_test.go @@ -77,7 +77,6 @@ var ( BluecatDNSDeployType: defaultConfig.BluecatDNSDeployType, BluecatSkipTLSVerify: false, CloudflareProxied: false, - CloudflareZonesPerPage: 50, CoreDNSPrefix: "/skydns/", AkamaiServiceConsumerDomain: "", AkamaiClientToken: "", @@ -182,7 +181,6 @@ var ( BluecatDNSDeployType: "full-deploy", BluecatSkipTLSVerify: true, CloudflareProxied: true, - CloudflareZonesPerPage: 20, CoreDNSPrefix: "/coredns/", AkamaiServiceConsumerDomain: "oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net", AkamaiClientToken: "o184671d5307a388180fbf7f11dbdf46", @@ -294,7 +292,6 @@ func TestParseFlags(t *testing.T) { "--bluecat-dns-deploy-type=full-deploy", "--bluecat-skip-tls-verify", "--cloudflare-proxied", - "--cloudflare-zones-per-page=20", "--coredns-prefix=/coredns/", "--akamai-serviceconsumerdomain=oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net", "--akamai-client-token=o184671d5307a388180fbf7f11dbdf46", @@ -417,7 +414,6 @@ func TestParseFlags(t *testing.T) { "EXTERNAL_DNS_BLUECAT_ROOT_ZONE": "arg", "EXTERNAL_DNS_BLUECAT_SKIP_TLS_VERIFY": "1", "EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1", - "EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE": "20", "EXTERNAL_DNS_COREDNS_PREFIX": "/coredns/", "EXTERNAL_DNS_AKAMAI_SERVICECONSUMERDOMAIN": "oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net", "EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN": "o184671d5307a388180fbf7f11dbdf46", diff --git a/provider/cloudflare/cloudflare.go b/provider/cloudflare/cloudflare.go index 6948be49b6..313b50017e 100644 --- a/provider/cloudflare/cloudflare.go +++ b/provider/cloudflare/cloudflare.go @@ -118,11 +118,10 @@ type CloudFlareProvider struct { provider.BaseProvider Client cloudFlareDNS // only consider hosted zones managing domains ending in this suffix - domainFilter endpoint.DomainFilter - zoneIDFilter provider.ZoneIDFilter - proxiedByDefault bool - DryRun bool - PaginationOptions cloudflare.PaginationOptions + domainFilter endpoint.DomainFilter + zoneIDFilter provider.ZoneIDFilter + proxiedByDefault bool + DryRun bool } // cloudFlareChange differentiates between ChangActions @@ -148,7 +147,7 @@ func getRecordParam[T RecordParamsTypes](cfc cloudFlareChange) T { } // NewCloudFlareProvider initializes a new CloudFlare DNS based Provider. -func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, zonesPerPage int, proxiedByDefault bool, dryRun bool) (*CloudFlareProvider, error) { +func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, proxiedByDefault bool, dryRun bool) (*CloudFlareProvider, error) { // initialize via chosen auth method and returns new API object var ( config *cloudflare.API @@ -169,10 +168,6 @@ func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter prov zoneIDFilter: zoneIDFilter, proxiedByDefault: proxiedByDefault, DryRun: dryRun, - PaginationOptions: cloudflare.PaginationOptions{ - PerPage: zonesPerPage, - Page: 1, - }, } return provider, nil } @@ -180,7 +175,6 @@ func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter prov // Zones returns the list of hosted zones. func (p *CloudFlareProvider) Zones(ctx context.Context) ([]cloudflare.Zone, error) { result := []cloudflare.Zone{} - p.PaginationOptions.Page = 1 // if there is a zoneIDfilter configured // && if the filter isn't just a blank string (used in tests) diff --git a/provider/cloudflare/cloudflare_test.go b/provider/cloudflare/cloudflare_test.go index b2136bff0a..7ac0f47f9e 100644 --- a/provider/cloudflare/cloudflare_test.go +++ b/provider/cloudflare/cloudflare_test.go @@ -640,7 +640,6 @@ func TestCloudflareProvider(t *testing.T) { _, err := NewCloudFlareProvider( endpoint.NewDomainFilter([]string{"bar.com"}), provider.NewZoneIDFilter([]string{""}), - 25, false, true) if err != nil { @@ -652,7 +651,6 @@ func TestCloudflareProvider(t *testing.T) { _, err = NewCloudFlareProvider( endpoint.NewDomainFilter([]string{"bar.com"}), provider.NewZoneIDFilter([]string{""}), - 1, false, true) if err != nil { @@ -663,7 +661,6 @@ func TestCloudflareProvider(t *testing.T) { _, err = NewCloudFlareProvider( endpoint.NewDomainFilter([]string{"bar.com"}), provider.NewZoneIDFilter([]string{""}), - 50, false, true) if err == nil { From 2cc75a04b755353a51e3ecdee21144410846088b Mon Sep 17 00:00:00 2001 From: Artur Rodrigues Date: Thu, 5 Jan 2023 16:00:58 +0000 Subject: [PATCH 2/4] Pass PerPage param to calls to ListDNSRecords Signed-off-by: Artur Rodrigues --- provider/cloudflare/cloudflare.go | 26 +++++++++++++++++-- provider/cloudflare/cloudflare_test.go | 35 ++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/provider/cloudflare/cloudflare.go b/provider/cloudflare/cloudflare.go index 313b50017e..bfb46fa6b3 100644 --- a/provider/cloudflare/cloudflare.go +++ b/provider/cloudflare/cloudflare.go @@ -60,6 +60,8 @@ var recordTypeProxyNotSupported = map[string]bool{ "SRV": true, } +var defaultListDNSRecordsPerPage int = 100 + // cloudFlareDNS is the subset of the CloudFlare API that we actually use. Add methods as required. Signatures must match exactly. type cloudFlareDNS interface { UserDetails(ctx context.Context) (cloudflare.User, error) @@ -223,7 +225,7 @@ func (p *CloudFlareProvider) Records(ctx context.Context) ([]*endpoint.Endpoint, endpoints := []*endpoint.Endpoint{} for _, zone := range zones { - records, _, err := p.Client.ListDNSRecords(ctx, cloudflare.ZoneIdentifier(zone.ID), cloudflare.ListDNSRecordsParams{}) + records, err := p.listDNSRecordsWithAutoPagination(ctx, zone.ID) if err != nil { return nil, err } @@ -297,7 +299,7 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud changesByZone := p.changesByZone(zones, changes) for zoneID, changes := range changesByZone { - records, _, err := p.Client.ListDNSRecords(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListDNSRecordsParams{}) + records, err := p.listDNSRecordsWithAutoPagination(ctx, zoneID) if err != nil { return fmt.Errorf("could not fetch records from zone, %v", err) } @@ -414,6 +416,26 @@ func (p *CloudFlareProvider) newCloudFlareChange(action string, endpoint *endpoi } } +// listDNSRecords performs automatic pagination of results on requests to cloudflare.ListDNSRecords with custom per_page values +func (p *CloudFlareProvider) listDNSRecordsWithAutoPagination(ctx context.Context, zoneID string) ([]cloudflare.DNSRecord, error) { + var records []cloudflare.DNSRecord + resultInfo := cloudflare.ResultInfo{PerPage: defaultListDNSRecordsPerPage, Page: 1} + params := cloudflare.ListDNSRecordsParams{ResultInfo: resultInfo} + for { + pageRecords, resultInfo, err := p.Client.ListDNSRecords(ctx, cloudflare.ZoneIdentifier(zoneID), params) + if err != nil { + return nil, err + } + + records = append(records, pageRecords...) + params.ResultInfo = resultInfo.Next() + if params.ResultInfo.Done() { + break + } + } + return records, nil +} + func shouldBeProxied(endpoint *endpoint.Endpoint, proxiedByDefault bool) bool { proxied := proxiedByDefault diff --git a/provider/cloudflare/cloudflare_test.go b/provider/cloudflare/cloudflare_test.go index 7ac0f47f9e..74d2e19611 100644 --- a/provider/cloudflare/cloudflare_test.go +++ b/provider/cloudflare/cloudflare_test.go @@ -20,6 +20,8 @@ import ( "context" "errors" "os" + "sort" + "strings" "testing" cloudflare "github.com/cloudflare/cloudflare-go" @@ -151,9 +153,36 @@ func (m *mockCloudFlareClient) ListDNSRecords(ctx context.Context, rc *cloudflar for _, record := range zone { result = append(result, record) } - return result, &cloudflare.ResultInfo{}, nil } - return result, &cloudflare.ResultInfo{}, nil + + if len(result) == 0 { + return result, &cloudflare.ResultInfo{Page: 1, TotalPages: 1, Count: 0, Total: 0}, nil + } + + // if not pagination options were passed in, return the result as is + if rp.Page == 0 && rp.PerPage == 0 { + return result, &cloudflare.ResultInfo{Page: 1, TotalPages: 1, Count: len(result), Total: len(result)}, nil + } + + // otherwise, split the result into chunks of size rp.PerPage to simulate the pagination from the API + chunks := [][]cloudflare.DNSRecord{} + + // to ensure consistency in the multiple calls to this function, sort the result slice + sort.Slice(result, func(i, j int) bool { return strings.Compare(result[i].ID, result[j].ID) > 0 }) + for rp.PerPage < len(result) { + result, chunks = result[rp.PerPage:], append(chunks, result[0:rp.PerPage]) + } + chunks = append(chunks, result) + + // return the requested page + partialResult := chunks[rp.Page-1] + return partialResult, &cloudflare.ResultInfo{ + PerPage: rp.PerPage, + Page: rp.Page, + TotalPages: len(chunks), + Count: len(partialResult), + Total: len(result), + }, nil } func (m *mockCloudFlareClient) UpdateDNSRecord(ctx context.Context, rc *cloudflare.ResourceContainer, rp cloudflare.UpdateDNSRecordParams) error { @@ -611,6 +640,8 @@ func TestCloudflareRecords(t *testing.T) { "001": ExampleDomain, }) + // change the defaultListDNSRecordsPerPage value to 1 test the pagination behaviour + defaultListDNSRecordsPerPage = 1 provider := &CloudFlareProvider{ Client: client, } From 789190d565b312775c07c62a60201b1500e2bb93 Mon Sep 17 00:00:00 2001 From: Artur Rodrigues Date: Thu, 5 Jan 2023 16:01:10 +0000 Subject: [PATCH 3/4] Parametize cloudflare dns records per page Signed-off-by: Artur Rodrigues --- main.go | 2 +- pkg/apis/externaldns/types.go | 3 +++ pkg/apis/externaldns/types_test.go | 4 ++++ provider/cloudflare/cloudflare.go | 26 +++++++++++++------------- provider/cloudflare/cloudflare_test.go | 19 +++++++++++-------- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/main.go b/main.go index 815afa0b01..6607a60645 100644 --- a/main.go +++ b/main.go @@ -233,7 +233,7 @@ func main() { case "civo": p, err = civo.NewCivoProvider(domainFilter, cfg.DryRun) case "cloudflare": - p, err = cloudflare.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun) + p, err = cloudflare.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun, cfg.CloudflareDNSRecordsPerPage) case "rcodezero": p, err = rcode0.NewRcodeZeroProvider(domainFilter, cfg.DryRun, cfg.RcodezeroTXTEncrypt) case "google": diff --git a/pkg/apis/externaldns/types.go b/pkg/apis/externaldns/types.go index 3bd9a61d70..a7fa2bd9c5 100644 --- a/pkg/apis/externaldns/types.go +++ b/pkg/apis/externaldns/types.go @@ -105,6 +105,7 @@ type Config struct { BluecatDNSDeployType string BluecatSkipTLSVerify bool CloudflareProxied bool + CloudflareDNSRecordsPerPage int CoreDNSPrefix string RcodezeroTXTEncrypt bool AkamaiServiceConsumerDomain string @@ -252,6 +253,7 @@ var defaultConfig = &Config{ BluecatConfigFile: "/etc/kubernetes/bluecat.json", BluecatDNSDeployType: "no-deploy", CloudflareProxied: false, + CloudflareDNSRecordsPerPage: 100, CoreDNSPrefix: "/skydns/", RcodezeroTXTEncrypt: false, AkamaiServiceConsumerDomain: "", @@ -470,6 +472,7 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("bluecat-dns-deploy-type", "When using the Bluecat provider, specify the type of DNS deployment to initiate after records are updated. Valid options are 'full-deploy' and 'no-deploy'. Deploy will only execute if --bluecat-dns-server-name is set (optional when --provider=bluecat)").Default(defaultConfig.BluecatDNSDeployType).StringVar(&cfg.BluecatDNSDeployType) app.Flag("cloudflare-proxied", "When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled)").BoolVar(&cfg.CloudflareProxied) + app.Flag("cloudflare-dns-records-per-page", "When using the Cloudflare provider, specify how many DNS records listed per page, max possible 5,000 (default: 100)").Default(strconv.Itoa(defaultConfig.CloudflareDNSRecordsPerPage)).IntVar(&cfg.CloudflareDNSRecordsPerPage) app.Flag("coredns-prefix", "When using the CoreDNS provider, specify the prefix name").Default(defaultConfig.CoreDNSPrefix).StringVar(&cfg.CoreDNSPrefix) app.Flag("akamai-serviceconsumerdomain", "When using the Akamai provider, specify the base URL (required when --provider=akamai and edgerc-path not specified)").Default(defaultConfig.AkamaiServiceConsumerDomain).StringVar(&cfg.AkamaiServiceConsumerDomain) app.Flag("akamai-client-token", "When using the Akamai provider, specify the client token (required when --provider=akamai and edgerc-path not specified)").Default(defaultConfig.AkamaiClientToken).StringVar(&cfg.AkamaiClientToken) diff --git a/pkg/apis/externaldns/types_test.go b/pkg/apis/externaldns/types_test.go index 24a699b38d..87647ee218 100644 --- a/pkg/apis/externaldns/types_test.go +++ b/pkg/apis/externaldns/types_test.go @@ -77,6 +77,7 @@ var ( BluecatDNSDeployType: defaultConfig.BluecatDNSDeployType, BluecatSkipTLSVerify: false, CloudflareProxied: false, + CloudflareDNSRecordsPerPage: 100, CoreDNSPrefix: "/skydns/", AkamaiServiceConsumerDomain: "", AkamaiClientToken: "", @@ -181,6 +182,7 @@ var ( BluecatDNSDeployType: "full-deploy", BluecatSkipTLSVerify: true, CloudflareProxied: true, + CloudflareDNSRecordsPerPage: 5000, CoreDNSPrefix: "/coredns/", AkamaiServiceConsumerDomain: "oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net", AkamaiClientToken: "o184671d5307a388180fbf7f11dbdf46", @@ -292,6 +294,7 @@ func TestParseFlags(t *testing.T) { "--bluecat-dns-deploy-type=full-deploy", "--bluecat-skip-tls-verify", "--cloudflare-proxied", + "--cloudflare-dns-records-per-page=5000", "--coredns-prefix=/coredns/", "--akamai-serviceconsumerdomain=oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net", "--akamai-client-token=o184671d5307a388180fbf7f11dbdf46", @@ -414,6 +417,7 @@ func TestParseFlags(t *testing.T) { "EXTERNAL_DNS_BLUECAT_ROOT_ZONE": "arg", "EXTERNAL_DNS_BLUECAT_SKIP_TLS_VERIFY": "1", "EXTERNAL_DNS_CLOUDFLARE_PROXIED": "1", + "EXTERNAL_DNS_CLOUDFLARE_DNS_RECORDS_PER_PAGE": "5000", "EXTERNAL_DNS_COREDNS_PREFIX": "/coredns/", "EXTERNAL_DNS_AKAMAI_SERVICECONSUMERDOMAIN": "oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net", "EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN": "o184671d5307a388180fbf7f11dbdf46", diff --git a/provider/cloudflare/cloudflare.go b/provider/cloudflare/cloudflare.go index bfb46fa6b3..0f7bdf0cc6 100644 --- a/provider/cloudflare/cloudflare.go +++ b/provider/cloudflare/cloudflare.go @@ -60,8 +60,6 @@ var recordTypeProxyNotSupported = map[string]bool{ "SRV": true, } -var defaultListDNSRecordsPerPage int = 100 - // cloudFlareDNS is the subset of the CloudFlare API that we actually use. Add methods as required. Signatures must match exactly. type cloudFlareDNS interface { UserDetails(ctx context.Context) (cloudflare.User, error) @@ -120,10 +118,11 @@ type CloudFlareProvider struct { provider.BaseProvider Client cloudFlareDNS // only consider hosted zones managing domains ending in this suffix - domainFilter endpoint.DomainFilter - zoneIDFilter provider.ZoneIDFilter - proxiedByDefault bool - DryRun bool + domainFilter endpoint.DomainFilter + zoneIDFilter provider.ZoneIDFilter + proxiedByDefault bool + DryRun bool + DNSRecordsPerPage int } // cloudFlareChange differentiates between ChangActions @@ -149,7 +148,7 @@ func getRecordParam[T RecordParamsTypes](cfc cloudFlareChange) T { } // NewCloudFlareProvider initializes a new CloudFlare DNS based Provider. -func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, proxiedByDefault bool, dryRun bool) (*CloudFlareProvider, error) { +func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter provider.ZoneIDFilter, proxiedByDefault bool, dryRun bool, dnsRecordsPerPage int) (*CloudFlareProvider, error) { // initialize via chosen auth method and returns new API object var ( config *cloudflare.API @@ -165,11 +164,12 @@ func NewCloudFlareProvider(domainFilter endpoint.DomainFilter, zoneIDFilter prov } provider := &CloudFlareProvider{ // Client: config, - Client: zoneService{config}, - domainFilter: domainFilter, - zoneIDFilter: zoneIDFilter, - proxiedByDefault: proxiedByDefault, - DryRun: dryRun, + Client: zoneService{config}, + domainFilter: domainFilter, + zoneIDFilter: zoneIDFilter, + proxiedByDefault: proxiedByDefault, + DryRun: dryRun, + DNSRecordsPerPage: dnsRecordsPerPage, } return provider, nil } @@ -419,7 +419,7 @@ func (p *CloudFlareProvider) newCloudFlareChange(action string, endpoint *endpoi // listDNSRecords performs automatic pagination of results on requests to cloudflare.ListDNSRecords with custom per_page values func (p *CloudFlareProvider) listDNSRecordsWithAutoPagination(ctx context.Context, zoneID string) ([]cloudflare.DNSRecord, error) { var records []cloudflare.DNSRecord - resultInfo := cloudflare.ResultInfo{PerPage: defaultListDNSRecordsPerPage, Page: 1} + resultInfo := cloudflare.ResultInfo{PerPage: p.DNSRecordsPerPage, Page: 1} params := cloudflare.ListDNSRecordsParams{ResultInfo: resultInfo} for { pageRecords, resultInfo, err := p.Client.ListDNSRecords(ctx, cloudflare.ZoneIdentifier(zoneID), params) diff --git a/provider/cloudflare/cloudflare_test.go b/provider/cloudflare/cloudflare_test.go index 74d2e19611..3755c8c9bc 100644 --- a/provider/cloudflare/cloudflare_test.go +++ b/provider/cloudflare/cloudflare_test.go @@ -155,12 +155,12 @@ func (m *mockCloudFlareClient) ListDNSRecords(ctx context.Context, rc *cloudflar } } - if len(result) == 0 { + if len(result) == 0 || rp.PerPage == 0 { return result, &cloudflare.ResultInfo{Page: 1, TotalPages: 1, Count: 0, Total: 0}, nil } // if not pagination options were passed in, return the result as is - if rp.Page == 0 && rp.PerPage == 0 { + if rp.Page == 0 { return result, &cloudflare.ResultInfo{Page: 1, TotalPages: 1, Count: len(result), Total: len(result)}, nil } @@ -640,10 +640,10 @@ func TestCloudflareRecords(t *testing.T) { "001": ExampleDomain, }) - // change the defaultListDNSRecordsPerPage value to 1 test the pagination behaviour - defaultListDNSRecordsPerPage = 1 + // Set DNSRecordsPerPage to 1 test the pagination behaviour provider := &CloudFlareProvider{ - Client: client, + Client: client, + DNSRecordsPerPage: 1, } ctx := context.Background() @@ -672,7 +672,8 @@ func TestCloudflareProvider(t *testing.T) { endpoint.NewDomainFilter([]string{"bar.com"}), provider.NewZoneIDFilter([]string{""}), false, - true) + true, + 5000) if err != nil { t.Errorf("should not fail, %s", err) } @@ -683,7 +684,8 @@ func TestCloudflareProvider(t *testing.T) { endpoint.NewDomainFilter([]string{"bar.com"}), provider.NewZoneIDFilter([]string{""}), false, - true) + true, + 5000) if err != nil { t.Errorf("should not fail, %s", err) } @@ -693,7 +695,8 @@ func TestCloudflareProvider(t *testing.T) { endpoint.NewDomainFilter([]string{"bar.com"}), provider.NewZoneIDFilter([]string{""}), false, - true) + true, + 5000) if err == nil { t.Errorf("expected to fail") } From d8bf24696e394d921be3ac114c908f261dfd3655 Mon Sep 17 00:00:00 2001 From: Artur Rodrigues Date: Thu, 5 Jan 2023 16:56:59 +0000 Subject: [PATCH 4/4] Document new option Signed-off-by: Artur Rodrigues --- docs/tutorials/cloudflare.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/cloudflare.md b/docs/tutorials/cloudflare.md index 6f03ba5dd7..e8483d94c8 100644 --- a/docs/tutorials/cloudflare.md +++ b/docs/tutorials/cloudflare.md @@ -18,13 +18,17 @@ Snippet from [Cloudflare - Getting Started](https://api.cloudflare.com/#getting- >The Cloudflare API is a RESTful API based on HTTPS requests and JSON responses. If you are registered with Cloudflare, you can obtain your API key from the bottom of the "My Account" page, found here: [Go to My account](https://dash.cloudflare.com/profile). -API Token will be preferred for authentication if `CF_API_TOKEN` environment variable is set. +API Token will be preferred for authentication if `CF_API_TOKEN` environment variable is set. Otherwise `CF_API_KEY` and `CF_API_EMAIL` should be set to run ExternalDNS with Cloudflare. When using API Token authentication, the token should be granted Zone `Read`, DNS `Edit` privileges, and access to `All zones`. If you would like to further restrict the API permissions to a specific zone (or zones), you also need to use the `--zone-id-filter` so that the underlying API requests only access the zones that you explicitly specify, as opposed to accessing all zones. +## Throttling + +Cloudflare API has a [global rate limit of 1,200 requests per five minutes](https://developers.cloudflare.com/fundamentals/api/reference/limits/). Running several fast polling ExternalDNS instances in a given account can easily hit that limit. The AWS Provider [docs](./aws.md#throttling) has some recommendations that can be followed here too, but in particular, consider passing `--cloudflare-dns-records-per-page` with a high value (maximum is 5,000). + ## Deploy ExternalDNS Connect your `kubectl` client to the cluster you want to test ExternalDNS with. @@ -57,6 +61,7 @@ spec: - --zone-id-filter=023e105f4ecef8ad9ca31a8372d0c353 # (optional) limit to a specific zone. - --provider=cloudflare - --cloudflare-proxied # (optional) enable the proxy feature of Cloudflare (DDOS protection, CDN...) + - --cloudflare-dns-records-per-page=5000 # (optional) configure how many DNS records to fetch per request env: - name: CF_API_KEY value: "YOUR_CLOUDFLARE_API_KEY" @@ -81,7 +86,7 @@ rules: resources: ["services","endpoints","pods"] verbs: ["get","watch","list"] - apiGroups: ["extensions","networking.k8s.io"] - resources: ["ingresses"] + resources: ["ingresses"] verbs: ["get","watch","list"] - apiGroups: [""] resources: ["nodes"] @@ -125,6 +130,7 @@ spec: - --zone-id-filter=023e105f4ecef8ad9ca31a8372d0c353 # (optional) limit to a specific zone. - --provider=cloudflare - --cloudflare-proxied # (optional) enable the proxy feature of Cloudflare (DDOS protection, CDN...) + - --cloudflare-dns-records-per-page=5000 # (optional) configure how many DNS records to fetch per request env: - name: CF_API_KEY value: "YOUR_CLOUDFLARE_API_KEY"