Working with Large Sets of Records in Sugar Bulk API

by Cristian Golopenta on October 27, 2016

When it comes to integrating an external application with Sugar API system, there are some things we need to consider, one of them being the performance optimizations, when linking the pieces together.

Sugar comes with a response to this concern, with the implementation of Bulk API.

What it does, is basically putting together a collection of Sugar API calls, then sending all of them at once to the server, instead of doing a separate request for each individual api. This way, by batching the requests, the time performance is significantly improved. The data transfer protocols and resources are taken into account only for 1 request, and that is why the execution time is reduced.

Here is some test data, to highlight the performance gains, in terms of execution time, when using Sugar Bulk. The results below were obtained after running different routine tests (creating/ retrieving/ updating/ deleting sugar  records) both by using sequential requests and by using Bulk. Each test is consisted of a series of the same type of http requests (either POST, GET, PUT or DELETE) on different amount of data.

Sugar 7.6.1.0 – Requests without Bulk
No. of recordsCREATERETRIEVEUPDATEDELETE
2 records1.15s0.93s1.10s1.23s
5 records3.25s2.09s2.72s3.04s
10 records5.53s4.69s5.01s5.75s
100 records55.57s43.89s53.96s60.33s
Sugar 7.6.1.0 – Requests with Bulk
No. of recordsCREATERETRIEVEUPDATEDELETE
2 records0.59s0.47s0.60s0.64s
5 records0.93s0.53s0.72s0.94s
10 records2.47s0.71s1.05s1.23s
100 records11.15s3.12s7.55s9.12s

It is clear that, when working with large sets of data, using Bulk can make a real difference.

Below there are some examples of how to use the Sugar Bulk API, with different request types: For every sample below, the variables used are:

  • $base_url = the base URL of the Sugar instance
  • $access_token = the token obtained after the Sugar login request

POST - for creating multiple 'contact' records

$headers = array(
	'OAuth-Token: '.$access_token
);
$bulk_data = array('requests' => array());
$contact_data = array(
	"deleted"=>false,
	"do_not_call"=>false,
	"portal_active"=>false,
	"mkto_sync"=>false,
	"gc_status_c"=>"",
	"gc_latitude_c"=>"220.00000000",
	"gc_longitude_c"=>"320.00000000",
	"gc_status_detail_c"=>"",
	"assigned_user_id"=>"1",
	"tag"=>array(),
	"preferred_language"=>"en_us",
	"salutation"=>"",
	"lead_source"=>"",
	"team_name"=>array(
		0=>array(
			"id"=>"1",
			"display_name"=>"Global",
			"name"=>"Global",
			"name_2"=>"",
			"primary"=>true
		)
	),
	"first_name"=>"John",
	"full_name"=>"John Doe",
	"last_name"=>"Doe",
	"title"=>"Segnor",
	"department"=>"department A",
	"email"=>array(
		0=>array(
			"email_address"=>"test@gmail.com",
			"primary_address"=>true
		)
	),
	"phone_mobile"=>"0777777777",
	"primary_address_street"=>"str",
	"alt_address_street"=>"str",
	"primary_address_city"=>"city",
	"alt_address_city"=>"city",
	"primary_address_state"=>"state",
	"alt_address_state"=>"state",
	"primary_address_postalcode"=>"1231231",
	"alt_address_postalcode"=>"1231231",
	"primary_address_country"=>"Romania",
	"alt_address_country"=>"Romania",
	"phone_fax"=>"2222",
	"twitter"=>"twitter acc",
	"description"=>"desc",
	"phone_work"=>"0666666666"
);

for($i=1; $i<=$contacts_number; $i++)
{
	$contact_data['first_name'] = 'test';
	$contact_data['last_name'] = 'contact_'.$i;
	$contact_data['full_name'] = $contact_data['first_name'].' '.$contact_data['last_name'];

		
	$bulk_request_data = array(
		'bulk' => '1',
		'contentType' => 'application/json',
		'dataType' => 'json',
		'headers' => $headers,
		'timeout' => 100000,
		'method' => 'POST',
		'url' => 'v10/Contacts',
		'data' => json_encode($contact_data),
	);
	$bulk_data['requests'][] = $bulk_request_data;
}

$url = $base_url.'/rest/v10/bulk'
$curl = curl_init();

$default_options = array(
	 CURLOPT_RETURNTRANSFER => true,
	 CURLOPT_SSL_VERIFYPEER => false,
	 CURLOPT_SSL_VERIFYHOST => false,
);
curl_setopt_array($curl, $default_options);
curl_setopt($curl, CURLOPT_POST, TRUE);
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
curl_setopt($curl, CURLOPT_URL, $url);

$curl_response = curl_exec($curl);
curl_close($curl);

GET - for retrieving multiple 'contact' records ($ids = list with records' IDs we want to retrieve)

$headers = array(
	'OAuth-Token: '.$access_token
);
$bulk_data = array('requests' => array());

foreach($ids as $id)
{	
	$bulk_request_data = array(
		'bulk' => '1',
		'contentType' => 'application/json',
		'dataType' => 'json',
		'headers' => $headers,
		'timeout' => 100000,
		'method' => 'GET',
		'url' => 'v10/Contacts/'.$id,
	);
	$bulk_data['requests'][] = $bulk_request_data;
}

$url = $base_url.'/rest/v10/bulk'
$curl = curl_init();

$default_options = array(
	 CURLOPT_RETURNTRANSFER => true,
	 CURLOPT_SSL_VERIFYPEER => false,
	 CURLOPT_SSL_VERIFYHOST => false,
);
curl_setopt_array($curl, $default_options);
curl_setopt($curl, CURLOPT_POST, TRUE);
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
curl_setopt($curl, CURLOPT_URL, $url);

$curl_response = curl_exec($curl);
curl_close($curl);

PUT - for updating a series of 'contact' records ($ids = list with records IDs we want to modify)

$headers = array(
	'OAuth-Token: '.$access_token
);
$bulk_data = array('requests' => array());
$contact_data = array('title'=>'modified');
foreach($ids as $id)
{	
	$bulk_request_data = array(
		'bulk' => '1',
		'contentType' => 'application/json',
		'dataType' => 'json',
		'headers' => $headers,
		'timeout' => 100000,
		'method' => 'PUT',
		'url' => 'v10/Contacts/'.$id,
		'data' => json_encode($contact_data),
	);
	$bulk_data['requests'][] = $bulk_request_data;	
}
$url = $base_url.'/rest/v10/bulk'
$curl = curl_init();

$default_options = array(
	 CURLOPT_RETURNTRANSFER => true,
	 CURLOPT_SSL_VERIFYPEER => false,
	 CURLOPT_SSL_VERIFYHOST => false,
);
curl_setopt_array($curl, $default_options);
curl_setopt($curl, CURLOPT_POST, TRUE);
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
curl_setopt($curl, CURLOPT_URL, $url);

$curl_response = curl_exec($curl);
curl_close($curl);

DELETE - for removing multiple 'contact' records ($ids = list with records IDs we want to delete)

$headers = array(
	'OAuth-Token: '.$access_token
);
$bulk_data = array('requests' => array());
foreach($ids as $id)
{
	$bulk_request_data = array(
		'bulk' => '1',
		'contentType' => 'application/json',
		'dataType' => 'json',
		'headers' => $headers,
		'timeout' => 100000,
		'method' => 'DELETE',
		'url' => 'v10/Contacts/'.$id,
	);
	$bulk_data['requests'][] = $bulk_request_data;
}
$url = $base_url.'/rest/v10/bulk'
$curl = curl_init();

$default_options = array(
	 CURLOPT_RETURNTRANSFER => true,
	 CURLOPT_SSL_VERIFYPEER => false,
	 CURLOPT_SSL_VERIFYHOST => false,
);
curl_setopt_array($curl, $default_options);
curl_setopt($curl, CURLOPT_POST, TRUE);
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
curl_setopt($curl, CURLOPT_URL, $url);

$curl_response = curl_exec($curl);
curl_close($curl);

This concludes our case for implementing Bulk API which cuts down on time and resources to handle large data transfers. Feel free to use any of these samples to try out Sugar Bulk API’s capabilities. For more Sugar developer related articles, click here.

Find similar articles in these categories:

PRODUCT: SugarCRM

AUDIENCE: Developers