Create a streaming pipeline definition

Creates a new streaming pipeline definition in the account and project

POST /v1/projects/{projectId}/streaming-pipelines

Create a streaming pipeline definition

cURL

curl --request POST \
  --url https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "agentId": "12345678-1234-1234-1234-123456789012",
  "name": "A Streaming Pipeline",
  "streamingSource": {
    "connection": {
      "host": "127.0.0.1",
      "password": {
        "secretName": "the-secret-name",
        "secretKey": "the-secret-key",
        "secretLocation": "the-secret-location"
      },
      "port": 8471,
      "username": "db2_streaming_user",
      "jdbcProperties": {
        "date format": "ymd"
      }
    },
    "tables": [
      {
        "schema": "SCHEMA_A",
        "table": "TABLE_A"
      }
    ]
  },
  "streamingTarget": {
    "accountKey": {
      "secretName": "the-secret-name",
      "secretKey": "the-secret-key",
      "secretLocation": "the-secret-location"
    },
    "accountName": "abs-account-name",
    "container": "streaming-container",
    "prefix": "streaming-prefix"
  },
  "advancedProperties": {
    "max.batch.size": 10000
  }
}
'

import requests

url = "https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines"

payload = {
    "agentId": "12345678-1234-1234-1234-123456789012",
    "name": "A Streaming Pipeline",
    "streamingSource": {
        "connection": {
            "host": "127.0.0.1",
            "password": {
                "secretName": "the-secret-name",
                "secretKey": "the-secret-key",
                "secretLocation": "the-secret-location"
            },
            "port": 8471,
            "username": "db2_streaming_user",
            "jdbcProperties": { "date format": "ymd" }
        },
        "tables": [
            {
                "schema": "SCHEMA_A",
                "table": "TABLE_A"
            }
        ]
    },
    "streamingTarget": {
        "accountKey": {
            "secretName": "the-secret-name",
            "secretKey": "the-secret-key",
            "secretLocation": "the-secret-location"
        },
        "accountName": "abs-account-name",
        "container": "streaming-container",
        "prefix": "streaming-prefix"
    },
    "advancedProperties": { "max.batch.size": 10000 }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    agentId: '12345678-1234-1234-1234-123456789012',
    name: 'A Streaming Pipeline',
    streamingSource: {
      connection: {
        host: '127.0.0.1',
        password: {
          secretName: 'the-secret-name',
          secretKey: 'the-secret-key',
          secretLocation: 'the-secret-location'
        },
        port: 8471,
        username: 'db2_streaming_user',
        jdbcProperties: {'date format': 'ymd'}
      },
      tables: [{schema: 'SCHEMA_A', table: 'TABLE_A'}]
    },
    streamingTarget: {
      accountKey: {
        secretName: 'the-secret-name',
        secretKey: 'the-secret-key',
        secretLocation: 'the-secret-location'
      },
      accountName: 'abs-account-name',
      container: 'streaming-container',
      prefix: 'streaming-prefix'
    },
    advancedProperties: {'max.batch.size': 10000}
  })
};

fetch('https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'agentId' => '12345678-1234-1234-1234-123456789012',
    'name' => 'A Streaming Pipeline',
    'streamingSource' => [
        'connection' => [
                'host' => '127.0.0.1',
                'password' => [
                                'secretName' => 'the-secret-name',
                                'secretKey' => 'the-secret-key',
                                'secretLocation' => 'the-secret-location'
                ],
                'port' => 8471,
                'username' => 'db2_streaming_user',
                'jdbcProperties' => [
                                'date format' => 'ymd'
                ]
        ],
        'tables' => [
                [
                                'schema' => 'SCHEMA_A',
                                'table' => 'TABLE_A'
                ]
        ]
    ],
    'streamingTarget' => [
        'accountKey' => [
                'secretName' => 'the-secret-name',
                'secretKey' => 'the-secret-key',
                'secretLocation' => 'the-secret-location'
        ],
        'accountName' => 'abs-account-name',
        'container' => 'streaming-container',
        'prefix' => 'streaming-prefix'
    ],
    'advancedProperties' => [
        'max.batch.size' => 10000
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines"

	payload := strings.NewReader("{\n  \"agentId\": \"12345678-1234-1234-1234-123456789012\",\n  \"name\": \"A Streaming Pipeline\",\n  \"streamingSource\": {\n    \"connection\": {\n      \"host\": \"127.0.0.1\",\n      \"password\": {\n        \"secretName\": \"the-secret-name\",\n        \"secretKey\": \"the-secret-key\",\n        \"secretLocation\": \"the-secret-location\"\n      },\n      \"port\": 8471,\n      \"username\": \"db2_streaming_user\",\n      \"jdbcProperties\": {\n        \"date format\": \"ymd\"\n      }\n    },\n    \"tables\": [\n      {\n        \"schema\": \"SCHEMA_A\",\n        \"table\": \"TABLE_A\"\n      }\n    ]\n  },\n  \"streamingTarget\": {\n    \"accountKey\": {\n      \"secretName\": \"the-secret-name\",\n      \"secretKey\": \"the-secret-key\",\n      \"secretLocation\": \"the-secret-location\"\n    },\n    \"accountName\": \"abs-account-name\",\n    \"container\": \"streaming-container\",\n    \"prefix\": \"streaming-prefix\"\n  },\n  \"advancedProperties\": {\n    \"max.batch.size\": 10000\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"agentId\": \"12345678-1234-1234-1234-123456789012\",\n  \"name\": \"A Streaming Pipeline\",\n  \"streamingSource\": {\n    \"connection\": {\n      \"host\": \"127.0.0.1\",\n      \"password\": {\n        \"secretName\": \"the-secret-name\",\n        \"secretKey\": \"the-secret-key\",\n        \"secretLocation\": \"the-secret-location\"\n      },\n      \"port\": 8471,\n      \"username\": \"db2_streaming_user\",\n      \"jdbcProperties\": {\n        \"date format\": \"ymd\"\n      }\n    },\n    \"tables\": [\n      {\n        \"schema\": \"SCHEMA_A\",\n        \"table\": \"TABLE_A\"\n      }\n    ]\n  },\n  \"streamingTarget\": {\n    \"accountKey\": {\n      \"secretName\": \"the-secret-name\",\n      \"secretKey\": \"the-secret-key\",\n      \"secretLocation\": \"the-secret-location\"\n    },\n    \"accountName\": \"abs-account-name\",\n    \"container\": \"streaming-container\",\n    \"prefix\": \"streaming-prefix\"\n  },\n  \"advancedProperties\": {\n    \"max.batch.size\": 10000\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://eu1.api.matillion.com/dpc/v1/projects/{projectId}/streaming-pipelines")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"agentId\": \"12345678-1234-1234-1234-123456789012\",\n  \"name\": \"A Streaming Pipeline\",\n  \"streamingSource\": {\n    \"connection\": {\n      \"host\": \"127.0.0.1\",\n      \"password\": {\n        \"secretName\": \"the-secret-name\",\n        \"secretKey\": \"the-secret-key\",\n        \"secretLocation\": \"the-secret-location\"\n      },\n      \"port\": 8471,\n      \"username\": \"db2_streaming_user\",\n      \"jdbcProperties\": {\n        \"date format\": \"ymd\"\n      }\n    },\n    \"tables\": [\n      {\n        \"schema\": \"SCHEMA_A\",\n        \"table\": \"TABLE_A\"\n      }\n    ]\n  },\n  \"streamingTarget\": {\n    \"accountKey\": {\n      \"secretName\": \"the-secret-name\",\n      \"secretKey\": \"the-secret-key\",\n      \"secretLocation\": \"the-secret-location\"\n    },\n    \"accountName\": \"abs-account-name\",\n    \"container\": \"streaming-container\",\n    \"prefix\": \"streaming-prefix\"\n  },\n  \"advancedProperties\": {\n    \"max.batch.size\": 10000\n  }\n}"

response = http.request(request)
puts response.read_body

201

{
  "advancedProperties": {
    "max.batch.size": 10000
  },
  "agentId": "12345678-1234-1234-1234-123456789012",
  "name": "A Streaming Pipeline",
  "projectId": "12345678-1234-1234-1234-123456789012",
  "streamingPipelineId": "12345678-1234-1234-1234-123456789012",
  "streamingSource": {
    "connection": {
      "host": "127.0.0.1",
      "password": {
        "secretName": "the-secret-name",
        "secretKey": "the-secret-key",
        "secretLocation": "the-secret-location"
      },
      "port": 8471,
      "username": "db2_streaming_user",
      "jdbcProperties": {
        "date format": "ymd"
      }
    },
    "tables": [
      {
        "schema": "SCHEMA_A",
        "table": "TABLE_A"
      }
    ]
  },
  "streamingTarget": {
    "accountKey": {
      "secretName": "the-secret-name",
      "secretKey": "the-secret-key",
      "secretLocation": "the-secret-location"
    },
    "accountName": "abs-account-name",
    "container": "streaming-container",
    "prefix": "streaming-prefix"
  }
}

{
  "detail": "<string>",
  "instance": "<string>",
  "status": 123,
  "title": "<string>",
  "type": "<string>",
  "violations": [
    "<string>"
  ]
}

{
  "detail": "<string>",
  "instance": "<string>",
  "status": 123,
  "title": "<string>",
  "type": "<string>",
  "violations": [
    "<string>"
  ]
}

{
  "detail": "<string>",
  "instance": "<string>",
  "status": 123,
  "title": "<string>",
  "type": "<string>",
  "violations": [
    "<string>"
  ]
}

{
  "detail": "<string>",
  "instance": "<string>",
  "status": 123,
  "title": "<string>",
  "type": "<string>",
  "violations": [
    "<string>"
  ]
}

{
  "detail": "<string>",
  "instance": "<string>",
  "status": 123,
  "title": "<string>",
  "type": "<string>",
  "violations": [
    "<string>"
  ]
}

Authorizations

Authorization · string · header · required

a valid bearer token

Path Parameters

projectId · string<uuid> · required

The ID of the project in which to create the streaming pipeline definition

Body

application/json

The details of the streaming pipeline definition to create

agentId · string<uuid> · required

The ID of the agent that will run the streaming pipeline

Example:

"12345678-1234-1234-1234-123456789012"

name · string · required

The name of the streaming pipeline

Minimum string length: 1

Example:

"A Streaming Pipeline"

streamingSource · object · required

The details of the streaming source

Option 1

Option 2

Option 3

Option 4

Option 5

Show child attributes

streamingTarget · object · required

The details of the streaming target

Option 1

Option 2

Option 3

Show child attributes

advancedProperties · object

Any additional properties for the streaming pipeline

Show child attributes

Example:

{ "max.batch.size": 10000 }

Response

201

application/json

Successfully created the streaming pipeline definition

advancedProperties · object

Any additional properties for the streaming pipeline

Show child attributes

Example:

{ "max.batch.size": 10000 }

agentId · string<uuid>

The ID of the agent that will run the streaming pipeline

Example:

"12345678-1234-1234-1234-123456789012"

name · string

The name of the streaming pipeline

Example:

"A Streaming Pipeline"

projectId · string<uuid>

The ID of the project that contains the streaming pipeline definition

Example:

"12345678-1234-1234-1234-123456789012"

streamingPipelineId · string<uuid>

The ID of the streaming pipeline definition

Example:

"12345678-1234-1234-1234-123456789012"

streamingSource · object

The details of the streaming source

Option 1

Option 2

Option 3

Option 4

Option 5

Show child attributes

streamingTarget · object

The details of the streaming target

Option 1

Option 2

Option 3

Show child attributes