AWS DMS

To Kinesis data format

The data to stream is json string. Here are some samples for different operations:

Control Data

sometimes there are some control data sent to stream

{
  "metadata": {
    "timestamp": "2021-02-18T23:09:09.509179Z",
    "record-type": "control",
    "operation": "create-table",
    "partition-key-type": "task-id",
    "schema-name": "",
    "table-name": "awsdms_apply_exceptions"
  }
}

Full Load

When DMS run full load

{
    "data": {
        "id": 315209,
        "created_by_user_id": 1,
        "ownership_id": 1,
        "name": "dsdsd dsdsdds",
        "first_name": "dsdsd",
        "last_name": "dsdsdds",
        "lead_type": "Hello Leads",
        "lead_status": "Contacted",
        "phone": "12345",
        "email": "[e-mail]",
        "country": "CB",
        "created_at": "2020-01-09T04:13:38Z",
        "updated_at": "2020-01-15T22:45:17Z",
        "deleted_at": "2020-01-15T22:45:17Z",
        "last_viewed_datetime": "2020-01-15T22:45:01Z",
        "description": "dshl asdjsakl haa hausaa s"
    },
    "metadata": {
        "timestamp": "2020-12-15T05:47:42.724458Z",
        "record-type": "data",
        "operation": "load",
        "partition-key-type": "primary-key",
        "schema-name": "lab_logix_crm_replica",
        "table-name": "acy_leads"
    }
}

Insert

{
    "data": {
        "id": 315233,
        "created_by_user_id": 36,
        "ownership_id": 5,
        "name": "sau tchy",
        "first_name": "sau",
        "last_name": "tchy",
        "lead_type": "Online Leads",
        "lead_status": "New Lead",
        "phone": "N/A",
        "email": "[e-mail]",
        "state": "Guangdong",
        "country": "China",
        "created_at": "2016-08-09T15:35:24Z",
        "updated_at": "2019-05-22T06:09:29Z",
        "last_activity_datetime": "2019-05-22T06:09:29Z",
        "last_modified_date": "2019-04-09T05:11:07Z",
        "last_modified_by_user_id": 1,
        "last_viewed_datetime": "2019-05-13T07:22:04Z",
        "lead_source": "Prime WL"
    },
    "metadata": {
        "timestamp": "2021-02-18T23:57:11.970360Z",
        "record-type": "data",
        "operation": "insert",
        "partition-key-type": "schema-table",
        "schema-name": "lab_logix_crm_replica",
        "table-name": "acy_leads",
        "transaction-id": 87978110091475
    }
}

Delete

{
    "data": {
        "id": 315232,
        "created_by_user_id": 36,
        "ownership_id": 5,
        "name": "sau tchy",
        "first_name": "sau",
        "last_name": "tchy",
        "lead_type": "Online Leads",
        "lead_status": "New Lead",
        "phone": "N/A",
        "email": "[e-mail]",
        "state": "Guangdong",
        "country": "China",
        "created_at": "2016-08-09T15:35:24Z",
        "updated_at": "2019-05-22T06:09:29Z",
        "last_activity_datetime": "2019-05-22T06:09:29Z",
        "last_modified_date": "2019-04-09T05:11:07Z",
        "last_modified_by_user_id": 1,
        "last_viewed_datetime": "2019-05-13T07:22:04Z",
        "lead_source": "Prime WL"
    },
    "metadata": {
        "timestamp": "2021-02-19T00:00:59.976396Z",
        "record-type": "data",
        "operation": "delete",
        "partition-key-type": "schema-table",
        "schema-name": "lab_logix_crm_replica",
        "table-name": "acy_leads",
        "transaction-id": 87982405058771
    }
}

Update

{
    "data": {
        "id": 296004,
        "created_by_user_id": 36,
        "ownership_id": 5,
        "name": "sau tchy",
        "first_name": "sau",
        "last_name": "Changed Name",
        "lead_type": "Online Leads",
        "lead_status": "New Lead",
        "phone": "N/A",
        "email": "[e-mail]",
        "state": "Guangdong",
        "country": "China",
        "created_at": "2016-08-09T15:35:24Z",
        "updated_at": "2019-05-22T06:09:29Z",
        "last_activity_datetime": "2019-05-22T06:09:29Z",
        "last_modified_date": "2019-04-09T05:11:07Z",
        "last_modified_by_user_id": 1,
        "last_viewed_datetime": "2019-05-13T07:22:04Z",
        "lead_source": "Prime WL"
    },
    "metadata": {
        "timestamp": "2021-02-19T00:05:28.496819Z",
        "record-type": "data",
        "operation": "update",
        "partition-key-type": "schema-table",
        "schema-name": "lab_logix_crm_replica",
        "table-name": "acy_leads",
        "transaction-id": 87986700026067
    }
}

"operation": "update"

"data": data AFTER update