Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update azure console tile #15820

Merged
merged 4 commits into from
Feb 5, 2024
Merged

Update azure console tile #15820

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5dea535
Save web console changes
georgew5656 Jan 31, 2024
53bb291
Working new input type
georgew5656 Jan 31, 2024
7a84f36
fix tests
georgew5656 Feb 1, 2024
4aff5c1
Fix merge conflicts
georgew5656 Feb 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 web-console/assets/azure.png → web-console/assets/azureStorage.png
View file
Open in desktop
File renamed without changes
2 changes: 1 addition & 1 deletion web-console/src/druid-models/external-config/external-config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ export function summarizeInputSource(inputSource: InputSource, multiline: boolea

case 's3':
case 'google':
case 'azure': {
case 'azureStorage': {
const possibleLines = inputSource.uris || inputSource.prefixes;
if (nonEmptyArray(possibleLines)) {
let lines: string[] = possibleLines;
Expand Down
38 changes: 27 additions & 11 deletions web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export type IngestionComboType =
| 'index_parallel:druid'
| 'index_parallel:inline'
| 'index_parallel:s3'
| 'index_parallel:azure'
| 'index_parallel:azureStorage'
| 'index_parallel:delta'
| 'index_parallel:google'
| 'index_parallel:hdfs';
Expand Down Expand Up @@ -142,7 +142,7 @@ export function getIngestionComboType(
case 'druid':
case 'inline':
case 's3':
case 'azure':
case 'azureStorage':
case 'google':
case 'hdfs':
return `${ioConfig.type}:${inputSource.type}` as IngestionComboType;
Expand Down Expand Up @@ -170,7 +170,7 @@ export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra): s
case 'index_parallel:s3':
return 'Amazon S3';

case 'index_parallel:azure':
case 'index_parallel:azureStorage':
return 'Azure Data Lake';

case 'index_parallel:delta':
Expand Down Expand Up @@ -231,7 +231,7 @@ export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): s
case 'index_parallel:s3':
return 'druid-s3-extensions';

case 'index_parallel:azure':
case 'index_parallel:azureStorage':
return 'druid-azure-extensions';

case 'index_parallel:delta':
Expand Down Expand Up @@ -530,7 +530,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
name: 'inputSource.type',
label: 'Source type',
type: 'string',
suggestions: ['local', 'http', 'inline', 'delta', 's3', 'azure', 'google', 'hdfs'],
suggestions: ['local', 'http', 'inline', 'delta', 's3', 'azureStorage', 'google', 'hdfs'],
info: (
<p>
Druid connects to raw data through{' '}
Expand Down Expand Up @@ -844,15 +844,15 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
},
];

case 'index_parallel:azure':
case 'index_parallel:azureStorage':
return [
inputSourceType,
{
name: 'inputSource.uris',
label: 'Azure URIs',
type: 'string-array',
placeholder:
'azure://your-container/some-file1.ext, azure://your-container/some-file2.ext',
'azureStorage://your-storage-account/your-container/some-file1.ext, azureStorage://your-storage-account/your-container/some-file2.ext',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
Expand All @@ -870,7 +870,8 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
name: 'inputSource.prefixes',
label: 'Azure prefixes',
type: 'string-array',
placeholder: 'azure://your-container/some-path1, azure://your-container/some-path2',
placeholder:
'azureStorage://your-storage-account/your-container/some-path1, azureStorage://your-storage-account/your-container/some-path2',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
Expand All @@ -885,7 +886,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
name: 'inputSource.objects',
label: 'Azure objects',
type: 'json',
placeholder: '{"bucket":"your-container", "path":"some-file.ext"}',
placeholder: '{"bucket":"your-storage-account", "path":"your-container/some-file.ext"}',
required: true,
defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
info: (
Expand All @@ -902,6 +903,21 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
),
},
inputSourceFilter,
{
name: 'inputSource.properties.sharedAccessStorageToken',
label: 'Shared Access Storage Token',
type: 'string',
placeholder: '(sas token)',
info: (
<>
<p>Shared Access Storage Token for this storage account.</p>
<p>
Note: Inlining the sas token into the ingestion spec can be dangerous as it might
appear in server log files and can be seen by anyone accessing this console.
</p>
</>
),
},
];

case 'index_parallel:google':
Expand Down Expand Up @@ -1163,7 +1179,7 @@ export function getIoConfigTuningFormFields(
switch (ingestionComboType) {
case 'index_parallel:http':
case 'index_parallel:s3':
case 'index_parallel:azure':
case 'index_parallel:azureStorage':
case 'index_parallel:google':
case 'index_parallel:delta':
case 'index_parallel:hdfs':
Expand Down Expand Up @@ -1440,7 +1456,7 @@ export function guessDataSourceNameFromInputSource(inputSource: InputSource): st
}

case 's3':
case 'azure':
case 'azureStorage':
case 'google': {
const actualPath = (inputSource.objects || EMPTY_ARRAY)[0];
const uriPath =
Expand Down
39 changes: 28 additions & 11 deletions web-console/src/druid-models/input-source/input-source.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export type InputSourceDesc =
};
}
| {
type: 'google' | 'azure';
type: 'google' | 'azureStorage';
uris?: string[];
prefixes?: string[];
objects?: { bucket: string; path: string }[];
Expand Down Expand Up @@ -154,7 +154,7 @@ export function issueWithInputSource(inputSource: InputSource | undefined): stri
return;

case 's3':
case 'azure':
case 'azureStorage':
case 'google':
if (
!nonEmptyArray(inputSource.uris) &&
Expand Down Expand Up @@ -188,7 +188,7 @@ const KNOWN_TYPES = [
'http',
'local',
's3',
'azure',
'azureStorage',
'delta',
'google',
'hdfs',
Expand Down Expand Up @@ -349,9 +349,10 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
name: 'uris',
label: 'Azure URIs',
type: 'string-array',
placeholder: 'azure://your-container/some-file1.ext, azure://your-container/some-file2.ext',
placeholder:
'azureStorage://your-storage-account/your-container/some-file1.ext, azureStorage://your-storage-account/your-container/some-file2.ext',
defined: inputSource =>
inputSource.type === 'azure' &&
inputSource.type === 'azureStorage' &&
!deepGet(inputSource, 'prefixes') &&
!deepGet(inputSource, 'objects'),
required: true,
Expand All @@ -369,9 +370,10 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
name: 'prefixes',
label: 'Azure prefixes',
type: 'string-array',
placeholder: 'azure://your-container/some-path1, azure://your-container/some-path2',
placeholder:
'azureStorage://your-storage-account/your-container/some-path1, azureStorage://your-storage-account/your-container/some-path2',
defined: inputSource =>
inputSource.type === 'azure' &&
inputSource.type === 'azureStorage' &&
!deepGet(inputSource, 'uris') &&
!deepGet(inputSource, 'objects'),
required: true,
Expand All @@ -386,8 +388,8 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
name: 'objects',
label: 'Azure objects',
type: 'json',
placeholder: '{"bucket":"your-container", "path":"some-file.ext"}',
defined: inputSource => inputSource.type === 'azure' && deepGet(inputSource, 'objects'),
placeholder: '{"bucket":"your-storage-account", "path":"your-container/some-file.ext"}',
defined: inputSource => inputSource.type === 'azureStorage' && deepGet(inputSource, 'objects'),
required: true,
info: (
<>
Expand All @@ -402,7 +404,22 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
</>
),
},

{
name: 'properties.sharedAccessStorageToken',
label: 'Shared Access Storage Token',
type: 'string',
placeholder: '(sas token)',
defined: inputSource => inputSource.type === 'azureStorage',
info: (
<>
<p>Shared Access Storage Token for this storage account.</p>
<p>
Note: Inlining the sas token into the ingestion spec can be dangerous as it might appear
in server log files and can be seen by anyone accessing this console.
</p>
</>
),
},
// google
{
name: 'uris',
Expand Down Expand Up @@ -469,7 +486,7 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
type: 'string',
suggestions: FILTER_SUGGESTIONS,
placeholder: '*',
defined: typeIsKnown(KNOWN_TYPES, 's3', 'azure', 'google'),
defined: typeIsKnown(KNOWN_TYPES, 's3', 'azureStorage', 'google'),
info: (
<p>
A wildcard filter for files. See{' '}
Expand Down
4 changes: 2 additions & 2 deletions web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,8 @@ exports[`LoadDataView matches snapshot batch 1`] = `
onClick={[Function]}
>
<img
alt="Ingestion tile for index_parallel:azure"
src="/some/base_url/assets/azure.png"
alt="Ingestion tile for index_parallel:azureStorage"
src="/some/base_url/assets/azurestorage.png"
/>
<p>
Azure Data Lake
Expand Down
6 changes: 3 additions & 3 deletions web-console/src/views/load-data-view/load-data-view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,7 +868,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
{mode !== 'streaming' && (
<>
{this.renderIngestionCard('index_parallel:s3')}
{this.renderIngestionCard('index_parallel:azure')}
{this.renderIngestionCard('index_parallel:azureStorage')}
{this.renderIngestionCard('index_parallel:google')}
{this.renderIngestionCard('index_parallel:delta')}
{this.renderIngestionCard('index_parallel:hdfs')}
Expand Down Expand Up @@ -962,7 +962,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
case 'index_parallel:s3':
return <p>Load text based, orc, or parquet data from Amazon S3.</p>;

case 'index_parallel:azure':
case 'index_parallel:azureStorage':
return <p>Load text based, orc, or parquet data from Azure.</p>;

case 'index_parallel:google':
Expand Down Expand Up @@ -1027,7 +1027,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
case 'index_parallel:druid':
case 'index_parallel:inline':
case 'index_parallel:s3':
case 'index_parallel:azure':
case 'index_parallel:azureStorage':
case 'index_parallel:google':
case 'index_parallel:delta':
case 'index_parallel:hdfs':
Expand Down
2 changes: 1 addition & 1 deletion web-console/src/views/workbench-view/input-source-step/input-source-info.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ export const InputSourceInfo = React.memo(function InputSourceInfo(props: InputS
case 's3':
return <p>Load text based, avro, orc, or parquet data from Amazon S3.</p>;

case 'azure':
case 'azureStorage':
return <p>Load text based, avro, orc, or parquet data from Azure.</p>;

case 'google':
Expand Down
2 changes: 1 addition & 1 deletion web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS
<div className="input-source-step">
<div className="ingestion-cards">
{renderIngestionCard('s3')}
{renderIngestionCard('azure')}
{renderIngestionCard('azureStorage')}
{renderIngestionCard('google')}
{renderIngestionCard('hdfs')}
{renderIngestionCard('http')}
Expand Down
Loading