mobile device apis

(Mobile) Device APIsJames Pearce

@jamespearcehttp://tripleodeon.com

What are Device APIs?

“Client-side APIs that enable the development of Web Applications that interact with device hardware”

...and other capabilities outside of the browser’s traditional remit

Why are they important?

• They’re not... ...if you’re happy with a web of documents

• They are... ... if you want the the web to be competitive with native app platforms

HistoryCurrent Work

Use Cases

http://www.wapforum.org/what/technical/wtai-30-apr-98.pdf

WTAI Make Call

<a  href="wtai://wp/mc;16505551234">    Call  us</a>

<a  href="call.wmls#makeCall()">    Call  us</a>

//  call.wmlsextern  function  makeCall()  {    WTAPublic.makeCall("16505551234");}

WTAI Call Control

<a  href="calls.wmls#setupCall()">Call  us!</a><a  href="calls.wmls#acceptCall()">Brring!</a>

<a  href="wtai://cc/sd;1*2*3456">Enter  code</a>

//  calls.wmlsextern  function  setupCall()  {    WTACallCont.setup("16505551234",  1);}extern  function  acceptCall()  {    WTACallCont.accept("1",  1);}

WTAI Messaging

//  sms.wmlsextern  function  sendMsg()  {    WTANetText.send("16505551234",  "WML  rocks");}

extern  function  readMsg(i)  {    var  sms  =  WTANetText.read(i);    return  WTANetText.getFieldValue(        sms,  "body"    );}

WTAI Contacts

//  contacts.wmlsextern  function  addFriend(i,  num,  name)  {    WTAPhoneBook.write(i,  num,  name);}

//  return  structs  of  contact  detailsextern  function  getFriendById(i)  {    return  WTAPhoneBook.read("i",  i);}extern  function  getFriendByName(name)  {    return  WTAPhoneBook.read("t",  name);}

I may be laboring the point

But that was 1998...

... and you still can’t do any of this with a contemporary mobile browser

http://www.omtp.org/1.11

BONDI

“enables web based content to access native device capability, intermediated through a robust, but flexible security framework”

Messaging

var  sms  =  bondi.messaging.createSMS({    to:  ["16505551234"],    body:  "Home  soon"});

var  mms  =  bondi.messaging.createMMS({    to:  ["16505551234"],    subject:  "I  will  be...",    body:  "...home  soon"});

Messaging

bondi.messaging.sendSMS(    function  ()  {},  //  success    function  ()  {},  //  error    sms,  true);

bondi.messaging.sendMMS(    function  ()  {},  //  success    function  ()  {},  //  error    mms,  true);

Messaging

bondi.messaging.subscribeToSMS(    function  ()  {},                      //  success    function  ()  {},                      //  error    function  ()  {                          //  listener        alert("They  replied!");    },  {                                      from:  "16505551234"          //  filter    },    true                                            //  exclusive);

Contacts

bondi.pim.contact.getAddressBooks(    function(books)  {  //  success        var  contact  =  books[0].createContact({            name:  'James  Pearce',            emails:  [{email:'j@tripleodeon.com'}],            phoneNumbers:  [{number:'16505551234'}]        });    },      function  ()  {}        //  error);

* Yes, it’s true: you can’t actually make or receive calls

Telephony

bondi.telephonyLog.findLogEntries(    function  ()  {},  //  success    function  ()  {},  //  error,    [        bondi.telephonyLog.MISSED_CALLS_FOLDER,        bondi.telephonyLog.RECEIVED_CALLS_FOLDER    ],    {        phoneNumber:  "16505551234"    });

App Launcher

bondi.applauncher.launchApplication(    function  ()  {},  //  success    function  ()  {},  //  error    "tel:16505551234");

//  mailto:...//  sms:...//  file:...

http://www.omtp.org/1.11/apis

On one hand...

• Security-conscious architecture

• Comprehensive API support

• Consistent API patterns

• Conformance specifications

• Reasonable test coverage

On the other...

bondi.*

deviceapis.*

http://specs.wacapps.net/

WAC 2.0

• Accelerometer

• Calendar

• Camera

• Contacts

• Core

• Device interaction

• Device status

• File system

• Geolocation

• Messaging

• Orientation

• Tasks

• Viewport

• Webview

Which is all awesome

Except...

http://www.w3.org/2009/dap/

Device APIs Working Group

“to create client-side APIs that enable the development of Web Applications and Web Widgets that interact with devices services such as Calendar, Contacts, Camera, etc”

Joint efforts with WebRTC Working Group, WebApps Working Group

W3C Device APIs

• Media Capture (HTML)

• Media Capture (gUM)

• Contact

• Messaging

• Battery Status

• Network Information

• Sensor

• Vibration

• Calendar

• Permissions

• Menu

• Gallery

• System info

• Media Stream

Public working draft, http://www.w3.org/TR/2011/WD-html-media-capture-20110414

Media Capture (HTML)

<input  type="file"  accept="image/*"  />

<!-­‐-­‐  

   The  accept  attribute  is  'a  hint'.

   Specification  suggests  that  it  can  be    used  to  offer  alternative  picker  UIs.

-­‐-­‐>

Brad Lassey, September 2008

Media Capture (HTML)

<input  type="file"  accept="image/*"    capture="camera"/>

<!-­‐-­‐  

   Again,  a  hint.  Valid  options  are:

       camera,  camcorder,  microphone,  filesystem

-­‐-­‐>

Media Capture (HTML)

<input  type="file"  accept="image/*"    capture="camera"  id="photo"/>

var  photo  =  document.getElementById('photo');photo.files[0].getFormatData(    function  (data)  {},  //  success    function  ()  {}            //  error);

//  file  is  a  MediaFile,  extending  File//  data  is  a  MediaFileData

...seems to be under discussion

MediaFileData

interface  MediaFileData  {        attribute  DOMString          codecs;        attribute  unsigned  long  bitrate;        attribute  unsigned  long  height;        attribute  unsigned  long  width;        attribute  float                  duration;};

Editor's draft, http://dev.w3.org/2011/webrtc/editor/getusermedia.html

Media capture (gUM)

navigator.getUserMedia(    {        audio:  true,        video:  false    },    function  (stream)  {},  //  success    function  ()  {}                //  error);

//  stream  is  a  LocalMediaStream

LocalMediaStream

var  recorder  =  stream.record();

//  recorder  is  a  MediaStreamRecorder

recorder.getRecordedData(    function  (file)  {}    //  success);

//  file  is  a  File

URL.createObjectURL(stream);

//  a  Blob  URI  for,  say  <video  src='...'  />    

Editor's draft, http://dev.w3.org/2009/dap/messaging/

Messaging

navigator.sendMessage(    "sms:16505551234?"  +        "body=JavaScript%20says%20hi",      [],                          //  attachments  as  File  array    function  ()  {},  //  success    function  ()  {}    //  error);

//  also  mms:  and  mailto:

//  oh...  were  you  looking  for  receipt?

Editor's draft, http://dev.w3.org/2009/dap/system-info/Sensors.html

Sensorsnavigator.findSensors().onsuccess  =  function()  {    this.result.forEach(function  (sensor)  {        //  ...    });};

var  sensor  =  new  SensorConnection('Temperature');sensor.addEventListener('sensordata',  function(e)  {    if(e.data  >  20.0)  {        window.console.log('Hawt');    }});sensor.startWatch({  interval:  2000  });

Sensors

• Temperature, ºC

• AmbientLight, Lux

• AmbientNoise, dbA

• MagneticField, uTesla

• Proximity, cm

• AtmPressure, kP

• RelHumidity, %

Editor's draft, http://w3c-test.org/dap/contacts/

Contacts

navigator.contacts.find(    [                                              //  fields        'name',          'emails'    ],      function  (contacts)  {},  //  success    function  ()  {},                  //  error    {                                              //  find  options          filter:  'James',            multiple:  true    });

Contactinterface  Contact  {        readonly  attribute  DOMString                              id;                          attribute  DOMString?                            displayName;                          attribute  ContactName?                        name;                          attribute  DOMString?                            nickname;                          attribute  ContactField[]?                  phoneNumbers;                          attribute  ContactField[]?                  emails;                          attribute  ContactAddress[]?              addresses;                          attribute  ContactField[]?                  ims;                          attribute  ContactOrganization[]?    organizations;                          attribute  Date?                                      birthday;                          attribute  DOMString?                            note;                          attribute  ContactField[]?                  photos;                          attribute  DOMString[]?                        categories;                          attribute  ContactField[]?                  urls;};

Cool, but...

“The ability to add and update contact information is not a function of the API provided in this specification.”

Holy Contacts Editing WTF

var  vcard  =      'BEGIN:VCARD\r\n'  +    'VERSION:3.0\r\n'  +    'UID:'  +  contact.id  +  '\r\n'  +          'N:Pearce;James\r\n'  +    'FN:James  Pearce\r\n'  +    'END:VCARD';

document.getElementById('vcard').href  =    'data:text/x-­‐vcard;charset=utf-­‐8,'  +    encodeURIComponent(vcard);  

https://wiki.mozilla.org/WebAPI

WebAPI

“We are aiming at providing all the necessary APIs to build a basic HTML5 phone experience within the next 3-6 months”

WebAPI

• Telephony

• Messaging

• Contacts

• Camera

• Network

• USB

• Embedded browser

• Battery

• Device Status

• Settings

• Mouse Lock

• Vibrator

Hopefully W3C++

• “Our proposal might or might not look like it for the moment”

http://www.youtube.com/watch?v=RuIQskGD3u0#t=19s

Telephony!

var  tcall  =    navigator.telephony.dial("16505551234");

tcall.readyState;    //  dialing,  ringing,  busy,  connecting,    //  connected,  disconnecting,  disconnected,      //  incoming

tcall.onconnected  =  function  ()  {};    //  onbusy,  ondisconnected  etc

tcall2.answer();

Contacts

“We need read/write”

USB

var  usb  =  new  MozUSBManager();

usb.addEventListener("attachdevice",      function  ()  {});usb.addEventListener("detachdevice",      function  ()  {});

usb.claimDevice({    deviceClass:  ...,    deviceSubClass:  ...});

We deserve Device APIs...

• As forward-looking as WTAI

• As well-structured as BONDI

• As comprehensive as WAC

• As respectable as W3C DAP

• As agile as WebAPI

And we’ve got...

PhoneGap

• Accelerometer

• Camera

• Capture

• Compass

• Connection

• Contacts

• Device

• Events

• File

• Geolocation

• Media

• Notification

• Storage

Use Cases

https://github.com/jamesgpearce/compios5

Useless Demos

http://vimeo.com/adactio/iosbug https://github.com/scottjehl/iOS-Orientationchange-Fix

Fixing layout bugs, FML

window.addEventListener("deviceorientation",    function  (e)  {        orientation  =  Math.abs(w.orientation);        rotation  =  Math.abs(e.gamma);        if  (rotation  >  8  &&  orientation  ===  0)  {            if  (enabled)  {                disableZoom();            }  else  {                restoreZoom();            }        }    },  false);

Camera

• Profile pictures

• Review sites

• Photo sharing

• Recognition

• Bar-codes

• Text

• Faces

https://github.com/rendro/vintageJS

Filters!

$('img').vintage({    vignette:  {        black:  0.8,        white:  0.2    },    noise:  20,    screen:  {        red:  12,        ...    },    desaturate:  0.05});

http://fhtr.org/JSARToolKit/

Augmented reality?

Contacts & calendaring

• Add business details to address book

• Web-based messaging

• Facial recognition again...

• Presence

• Context-sensitivity of content

Messaging

• Sharing

• Notifications

• Offline transport fallback

• Frictionless web sign-on

Expect more

I still want...

• Audio processing

• Bluetooth

• RFID & NFC

• Both cameras

• Memory & power

• Native recognition

PS: does not work

window.addEventListener(    'appointmentdue',    function(appt)  {        if  (device.near(WORK))  {            siri.remind(                contacts.get('Peter'),                calendar.getDetails(appt)            );        }    },    false);

James Pearce@jamespearce

http://tripleodeon.com