Extraction phases — DevRev

Each snap-in must handle all the phases of Airdrop extraction. In a snap-in, you typically define a run function that iterates over events and invokes workers per extraction phase.

The Airdrop snap-in extraction lifecycle consists of four phases:

External sync units extraction (only for initial sync)
Metadata extraction
Data extraction
Attachments extraction

Each phase is defined in a separate file and is responsible for fetching the respective data.

Snap-in development is an iterative process. It typically begins with retrieving some data from the external system. The next step involves crafting an initial version of the external domain metadata and validating it through chef-cli. This metadata is used to prepare the initial domain mapping and checking for any possible issues. API calls to the external system are then corrected to fetch the missing data. Start by working with one item type (we recommend starting with users), and once it maps well to DevRev objects and imports as desired, proceed with other item types.

The SDK library exports a processTask function, which takes an object parameter with two keys:

task: a function that implements the functionality for the given phase.
onTimeout: a function that handles timeouts, typically by simply emitting a message to the Airdrop platform.

State management is crucial for snap-ins to maintain the state of the extraction task. State is saved to the Airdrop backend by calling the postState function. During the extraction the state is stored in the adapter and can be retrieved using the adapter.state property.

1 import { AirdropEvent, EventType, spawn } from "@devrev/ts-adaas";
2 
3 export interface ExtractorState {
4   todos: { completed: boolean };
5   users: { completed: boolean };
6   attachments: { completed: boolean };
7 }
8 
9 export const initialState: ExtractorState = {
10   todos: { completed: false },
11   users: { completed: false },
12   attachments: { completed: false },
13 };
14 
15 function getWorkerPerExtractionPhase(event: AirdropEvent) {
16   let path;
17   switch (event.payload.event_type) {
18     case EventType.ExtractionExternalSyncUnitsStart:
19       path = __dirname + '/workers/external-sync-units-extraction';
20       break;
21     case EventType.ExtractionMetadataStart:
22       path = __dirname + '/workers/metadata-extraction';
23       break;
24     case EventType.ExtractionDataStart:
25     case EventType.ExtractionDataContinue:
26       path = __dirname + '/workers/data-extraction';
27       break;
28     case EventType.ExtractionAttachmentsStart:
29     case EventType.ExtractionAttachmentsContinue:
30       path = __dirname + '/workers/attachments-extraction';
31       break;
32   }
33   return path;
34 }
35 
36 const run = async (events: AirdropEvent[]) => {
37   for (const event of events) {
38     const file = getWorkerPerExtractionPhase(event);
39     await spawn<ExtractorState>({
40       event,
41       initialState,
42       workerPath: file,
43       // options: {},
44     });
45   }
46 };
47 
48 export default run;

1	import { AirdropEvent, EventType, spawn } from "@devrev/ts-adaas";
2
3	export interface ExtractorState {
4	todos: { completed: boolean };
5	users: { completed: boolean };
6	attachments: { completed: boolean };
7	}
8
9	export const initialState: ExtractorState = {
10	todos: { completed: false },
11	users: { completed: false },
12	attachments: { completed: false },
13	};
14
15	function getWorkerPerExtractionPhase(event: AirdropEvent) {
16	let path;
17	switch (event.payload.event_type) {
18	case EventType.ExtractionExternalSyncUnitsStart:
19	path = __dirname + '/workers/external-sync-units-extraction';
20	break;
21	case EventType.ExtractionMetadataStart:
22	path = __dirname + '/workers/metadata-extraction';
23	break;
24	case EventType.ExtractionDataStart:
25	case EventType.ExtractionDataContinue:
26	path = __dirname + '/workers/data-extraction';
27	break;
28	case EventType.ExtractionAttachmentsStart:
29	case EventType.ExtractionAttachmentsContinue:
30	path = __dirname + '/workers/attachments-extraction';
31	break;
32	}
33	return path;
34	}
35
36	const run = async (events: AirdropEvent[]) => {
37	for (const event of events) {
38	const file = getWorkerPerExtractionPhase(event);
39	await spawn<ExtractorState>({
40	event,
41	initialState,
42	workerPath: file,
43	// options: {},
44	});
45	}
46	};
47
48	export default run;