Support (800) 775-8378 Contact Us

    Working with Large Sets of Records in Sugar Bulk API

    By Cristian Golopenta • October 27th, 2016
    Audiences: Developers

    When it comes to integrating an external application with Sugar API system, there are some things we need to consider, one of them being the performance optimizations, when linking the pieces together.

    Sugar comes with a response to this concern, with the implementation of Bulk API.

    What it does, is basically putting together a collection of Sugar API calls, then sending all of them at once to the server, instead of doing a separate request for each individual api. This way, by batching the requests, the time performance is significantly improved. The data transfer protocols and resources are taken into account only for 1 request, and that is why the execution time is reduced.

    Here is some test data, to highlight the performance gains, in terms of execution time, when using Sugar Bulk. The results below were obtained after running different routine tests (creating/ retrieving/ updating/ deleting sugar records records) both by using sequential requests and by using Bulk. Each test is consisted of a series of the same type of http requests (either POST, GET, PUT or DELETE) on different amount of data.

    Sugar 7.6.1.0 – Requests without Bulk
    No. of recordsCREATERETRIEVEUPDATEDELETE
    2 records1.15s0.93s1.10s1.23s
    5 records3.25s2.09s2.72s3.04s
    10 records5.53s4.69s5.01s5.75s
    100 records55.57s43.89s53.96s60.33s
    Sugar 7.6.1.0 – Requests with Bulk
    No. of recordsCREATERETRIEVEUPDATEDELETE
    2 records0.59s0.47s0.60s0.64s
    5 records0.93s0.53s0.72s0.94s
    10 records2.47s0.71s1.05s1.23s
    100 records11.15s3.12s7.55s9.12s

    It is clear that, when working with large sets of data, using Bulk can make a real difference.

    Below there are some examples of how to use the Sugar Bulk API, with different request types: For every sample below, the variables used are:

    • $base_url = the base URL of the Sugar instance
    • $access_token = the token obtained after the Sugar login request

    POST - for creating multiple 'contact' records

    $headers = array(
    	'OAuth-Token: '.$access_token
    );
    $bulk_data = array('requests' => array());
    $contact_data = array(
    	"deleted"=>false,
    	"do_not_call"=>false,
    	"portal_active"=>false,
    	"mkto_sync"=>false,
    	"gc_status_c"=>"",
    	"gc_latitude_c"=>"220.00000000",
    	"gc_longitude_c"=>"320.00000000",
    	"gc_status_detail_c"=>"",
    	"assigned_user_id"=>"1",
    	"tag"=>array(),
    	"preferred_language"=>"en_us",
    	"salutation"=>"",
    	"lead_source"=>"",
    	"team_name"=>array(
    		0=>array(
    			"id"=>"1",
    			"display_name"=>"Global",
    			"name"=>"Global",
    			"name_2"=>"",
    			"primary"=>true
    		)
    	),
    	"first_name"=>"John",
    	"full_name"=>"John Doe",
    	"last_name"=>"Doe",
    	"title"=>"Segnor",
    	"department"=>"department A",
    	"email"=>array(
    		0=>array(
    			"email_address"=>"test@gmail.com",
    			"primary_address"=>true
    		)
    	),
    	"phone_mobile"=>"0777777777",
    	"primary_address_street"=>"str",
    	"alt_address_street"=>"str",
    	"primary_address_city"=>"city",
    	"alt_address_city"=>"city",
    	"primary_address_state"=>"state",
    	"alt_address_state"=>"state",
    	"primary_address_postalcode"=>"1231231",
    	"alt_address_postalcode"=>"1231231",
    	"primary_address_country"=>"Romania",
    	"alt_address_country"=>"Romania",
    	"phone_fax"=>"2222",
    	"twitter"=>"twitter acc",
    	"description"=>"desc",
    	"phone_work"=>"0666666666"
    );
    
    for($i=1; $i<=$contacts_number; $i++)
    {
    	$contact_data['first_name'] = 'test';
    	$contact_data['last_name'] = 'contact_'.$i;
    	$contact_data['full_name'] = $contact_data['first_name'].' '.$contact_data['last_name'];
    
    		
    	$bulk_request_data = array(
    		'bulk' => '1',
    		'contentType' => 'application/json',
    		'dataType' => 'json',
    		'headers' => $headers,
    		'timeout' => 100000,
    		'method' => 'POST',
    		'url' => 'v10/Contacts',
    		'data' => json_encode($contact_data),
    	);
    	$bulk_data['requests'][] = $bulk_request_data;
    }
    
    $url = $base_url.'/rest/v10/bulk'
    $curl = curl_init();
    
    $default_options = array(
    	 CURLOPT_RETURNTRANSFER => true,
    	 CURLOPT_SSL_VERIFYPEER => false,
    	 CURLOPT_SSL_VERIFYHOST => false,
    );
    curl_setopt_array($curl, $default_options);
    curl_setopt($curl, CURLOPT_POST, TRUE);
    curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
    curl_setopt($curl, CURLOPT_URL, $url);
    
    $curl_response = curl_exec($curl);
    curl_close($curl);

    GET - for retrieving multiple 'contact' records ($ids = list with records' IDs we want to retrieve)

    $headers = array(
    	'OAuth-Token: '.$access_token
    );
    $bulk_data = array('requests' => array());
    
    foreach($ids as $id)
    {	
    	$bulk_request_data = array(
    		'bulk' => '1',
    		'contentType' => 'application/json',
    		'dataType' => 'json',
    		'headers' => $headers,
    		'timeout' => 100000,
    		'method' => 'GET',
    		'url' => 'v10/Contacts/'.$id,
    	);
    	$bulk_data['requests'][] = $bulk_request_data;
    }
    
    $url = $base_url.'/rest/v10/bulk'
    $curl = curl_init();
    
    $default_options = array(
    	 CURLOPT_RETURNTRANSFER => true,
    	 CURLOPT_SSL_VERIFYPEER => false,
    	 CURLOPT_SSL_VERIFYHOST => false,
    );
    curl_setopt_array($curl, $default_options);
    curl_setopt($curl, CURLOPT_POST, TRUE);
    curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
    curl_setopt($curl, CURLOPT_URL, $url);
    
    $curl_response = curl_exec($curl);
    curl_close($curl);

    PUT - for updating a series of 'contact' records ($ids = list with records IDs we want to modify)

    $headers = array(
    	'OAuth-Token: '.$access_token
    );
    $bulk_data = array('requests' => array());
    $contact_data = array('title'=>'modified');
    foreach($ids as $id)
    {	
    	$bulk_request_data = array(
    		'bulk' => '1',
    		'contentType' => 'application/json',
    		'dataType' => 'json',
    		'headers' => $headers,
    		'timeout' => 100000,
    		'method' => 'PUT',
    		'url' => 'v10/Contacts/'.$id,
    		'data' => json_encode($contact_data),
    	);
    	$bulk_data['requests'][] = $bulk_request_data;	
    }
    $url = $base_url.'/rest/v10/bulk'
    $curl = curl_init();
    
    $default_options = array(
    	 CURLOPT_RETURNTRANSFER => true,
    	 CURLOPT_SSL_VERIFYPEER => false,
    	 CURLOPT_SSL_VERIFYHOST => false,
    );
    curl_setopt_array($curl, $default_options);
    curl_setopt($curl, CURLOPT_POST, TRUE);
    curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
    curl_setopt($curl, CURLOPT_URL, $url);
    
    $curl_response = curl_exec($curl);
    curl_close($curl);

    DELETE - for removing multiple 'contact' records ($ids = list with records IDs we want to delete)

    $headers = array(
    	'OAuth-Token: '.$access_token
    );
    $bulk_data = array('requests' => array());
    foreach($ids as $id)
    {
    	$bulk_request_data = array(
    		'bulk' => '1',
    		'contentType' => 'application/json',
    		'dataType' => 'json',
    		'headers' => $headers,
    		'timeout' => 100000,
    		'method' => 'DELETE',
    		'url' => 'v10/Contacts/'.$id,
    	);
    	$bulk_data['requests'][] = $bulk_request_data;
    }
    $url = $base_url.'/rest/v10/bulk'
    $curl = curl_init();
    
    $default_options = array(
    	 CURLOPT_RETURNTRANSFER => true,
    	 CURLOPT_SSL_VERIFYPEER => false,
    	 CURLOPT_SSL_VERIFYHOST => false,
    );
    curl_setopt_array($curl, $default_options);
    curl_setopt($curl, CURLOPT_POST, TRUE);
    curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($bulk_data));
    curl_setopt($curl, CURLOPT_URL, $url);
    
    $curl_response = curl_exec($curl);
    curl_close($curl);

    This concludes our case for implementing Bulk API which cuts down on time and resources to handle large data transfers. Feel free to use any of these samples to try out Sugar Bulk API’s capabilities. For more Sugar developer related articles, click here.

    More From This Author
    Cristian Golopenta
    Developer at UpCurve Cloud