concurrent php in the etsy api
DESCRIPTION
How we at Etsy are adding concurrent data access to our PHP API. From PHP Day 2014TRANSCRIPT
Concurrent PHPin the Etsy API
Matthew Graham@lapsu
@EtsyAPI Lead#phpday2014 April
@lapsu@EtsyAPI
@lapsu@EtsyAPI
$1.3 BillionThings That Matter
@lapsu@EtsyAPI
@lapsu
Etsy's PHP
www api adminqueues cron
@EtsyAPI
@lapsu
~200 engineers
30+ deploys / day
@EtsyAPI
Also, Rasmus
@lapsu@EtsyAPI
Spoilers
@lapsu
Mobile Clients Are Special
1 Thread != No Concurrency
@EtsyAPI
@lapsu
<motivation>
@EtsyAPI
Premise:
@lapsu
The Future is Mobile
@EtsyAPI
@lapsu
Past The Future is Mobile
November 2013
@EtsyAPI
@lapsu@EtsyAPI
@EtsyAPI
Mobile Networks Suck
@lapsu
<
@EtsyAPI
Not Mobile
@lapsu
www.etsy.com/shop/AVintageWanderer
@EtsyAPI
Network Performance
@lapsu
3G < 4G
@EtsyAPI
Network Coverage
@lapsu
3G > 4G
@EtsyAPI
Mobile Requests
@lapsu
More != Better
@lapsu@EtsyAPI
@lapsu
1000ms Time To Glass
@EtsyAPI
@lapsu
1000ms- 900ms
-------------100ms
Network/Client------------------------ Server
@EtsyAPI
@lapsu
100ms = Bespoke + Concurrent
@EtsyAPI
@lapsu
Single Threads
@EtsyAPI
Concurrency
@lapsu@EtsyAPI
Main “Thread”
Child “Thread” Child “Thread”
@lapsu
</motivation><interface>
@EtsyAPI
Paul goes to Netflix
@lapsu@EtsyAPI
1 View : 1 Bespoke
@lapsu@EtsyAPI
ClientView
Bespoke
View View
Bespoke BespokeAPI
Multiple Clients
@lapsu@EtsyAPI
ClientsView
Bespoke
View View
Bespoke BespokeAPI
Bespoke : Components
@lapsu@EtsyAPI
Bespoke Bespoke BespokeAPI
Item User Shop Favs Tx
Components as REST
@lapsu@EtsyAPI
Bespoke BespokeAPI
Item User Shop Favs Tx
@lapsu
?includes=User
@EtsyAPI
Android User View
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
Concurrent Client
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
Making Requests
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
@lapsu@EtsyAPI
shop
favs
t0 t1
Inputs
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
Future Parameters
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
@lapsu@EtsyAPI
shop
favs
items
t0 t1 t2 t3 t4
~6 Lines
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
@lapsu
</interface><performance>
@EtsyAPI
@lapsu
Web Pages Are Clients Too
@EtsyAPI
Web First
@lapsu@EtsyAPI
API First
@lapsu@EtsyAPI
Android User View
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id);
Curl_Orchestrator::run( [$shop,$favs,$items]); return [$favs,$items];}
@EtsyAPI
Desktop Web User View
@lapsu
function handle($cli, $inp) { $shop = $cli->shop($inp->user_id); $favs = $cli->favs($inp->user_id); $items = $cli->items($shop->shop_id); $teams = $cli->teams($inp->user_id);
Curl_Orchestrator::run( [$shop,$favs,$items,$teams]); return [$favs,$items,$teams];}
@EtsyAPI
@lapsu
Activity Feed
@EtsyAPI
@lapsu
Activity FeedPage
@EtsyAPI
HomePage
30s of HTTP Time
@lapsu@EtsyAPI
30s of HTTP Time
@lapsu
880ms Real Time
@EtsyAPI
Components
@lapsu@EtsyAPI
Bespoke Bespoke Bespoke
API
Item User Shop Favs Tx
Components Cache
@lapsu@EtsyAPI
Bespoke Bespoke Bespoke
CacheAPI
Item User Shop Favs Tx
Local Call
@lapsu@EtsyAPI
Me
Long Distance Call
@lapsu@EtsyAPI
AtlanticOcean User
Long Distance Call
@lapsu@EtsyAPI
AtlanticOcean
API
Templates
User
@lapsu
</performance><internals>
@EtsyAPI
@lapsu
curl_multi_*
@EtsyAPI
@lapsu@EtsyAPI
curl?
General Sequence
@lapsu@EtsyAPI
curl_multi_init();curl_multi_add();curl_multi_exec();
while (!$done) { curl_multi_select(); curl_multi_exec(); curl_multi_info_read();}
curl_multi_init
@lapsu@EtsyAPI
$mh = curl_multi_init();
curl_multi_add_handle
@lapsu@EtsyAPI
$mh = curl_multi_init();$ch = curl_init($url);curl_setopt_array($ch, $options);curl_multi_add_handle($mh, $ch);
@lapsu@EtsyAPI
multi handle
handle handle handle
curl_multi_exec
@lapsu@EtsyAPI
do { $code = curl_multi_exec($mh, $r);} while ($code == CURLM_CALL_MULTI_PERFORM);
curl_multi_select
@lapsu@EtsyAPI
$cnt = curl_multi_select($mh, $tmout);
curl_multi_info_read
@lapsu@EtsyAPI
$info = curl_multi_info_read($mh);$ch = $info['handle'];$content = curl_multi_getcontent($ch);
@lapsu@EtsyAPI
localhost: Expected
@lapsu@EtsyAPI
R1
R2
R3
t0
Network: Actual
@lapsu@EtsyAPI
R2
R3
t0 t1
R1
curl Protocols
@lapsu@EtsyAPI
HTTP LDAP
Gopher POP3 IMAP
TELNET TFTP
curl_get_multi_handle_state
@lapsu@EtsyAPI
$state = CURLM_STATE_FIRST;curl_multi_get_handle_state( $mh, $ch, $state);$sent = $state > CURLM_STATE_FIRST && $state < CURLM_STATE_PERFORM;
Expected, Actual
@lapsu@EtsyAPI
R1
R2
R3
t0
Revised Sequence
@lapsu@EtsyAPI
curl_multi_init();curl_multi_add();while (!$sent) { curl_multi_exec(); curl_multi_get_handle_state();}
while (!$done) { curl_multi_select(); curl_multi_exec(); curl_multi_info_read();}
Still Headed Upstream
@lapsu@EtsyAPI
Patch URL
@lapsu
bit.ly/etsy_curl_multi_patch
@EtsyAPI
Recursion?
@lapsu@EtsyAPI
Recursion?
@lapsu
No.
@EtsyAPI
Recursion?
@lapsu
No?Not yet.
@EtsyAPI
Visibility
@lapsu@EtsyAPI
@lapsu
PHP Coroutines
@EtsyAPI
More Code
@lapsu
Available Upon Request
@EtsyAPI
@lapsu
codeascraft.etsy.com
@EtsyAPI
@lapsu
</internals><wrap/>
@EtsyAPI
@lapsu
Address Mobile Challenges
@EtsyAPI
@lapsu
PHP Does Concurrency
@EtsyAPI
PHP Abides
@lapsu@EtsyAPI
Concurrent PHP
in the Etsy APIMatthew Graham
@lapsu@EtsyAPI Lead
#phpday2014 April
Thank You
Reminder:
@lapsu
Repeat the questions
@EtsyAPI
@lapsu
SPDY / HTTP 2.0
@EtsyAPI