Hi Eric,

Thanks for the pointers.

`bosh vms` -- all running

Only 1 api vm running. cloud_controller_ng.log is almost constantly being
updated.

Below is the 500 error capture:

{"timestamp":1449752019.6870825,"message":"desire.app.request","log_level":"info","source":"cc.nsync.listener.client","data":{"request_guid":"e046d57f-277a-4f81-452c-3b6fb892432e::53557eba-6212-4727-9b43-ebbb03426b76","process_guid":"9f528159-1a7b-4876-92c9-34d040e9824d-29fd370c-04fd-4481-b432-39431460a963"},"thread_id":69975079694320,"fiber_id":69975079888800,"process_id":6524,"file":"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/diego/nsync_client.rb","lineno":15,"method":"desire_app"}

{"timestamp":1449752019.6899576,"message":"Cannot communicate with diego -
tried to send
start","log_level":"error","source":"cc.diego.runner","data":{"request_guid":"e046d57f-277a-4f81-452c-3b6fb892432e::53557eba-6212-4727-9b43-ebbb03426b76"},"thread_id":69975079694320,"fiber_id":69975079888800,"process_id":6524,"file":"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/diego/runner.rb","lineno":43,"method":"rescue
in with_logging"}

{"timestamp":1449752019.6909509,"message":"Request failed: 500:
{\"code\"=>10001, \"description\"=>\"getaddrinfo: Name or service not
known\", \"error_code\"=>\"CF-CannotCommunicateWithDiegoError\",
\"backtrace\"=>[\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/diego/runner.rb:44:in
`rescue in with_logging'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/diego/runner.rb:40:in
`with_logging'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/diego/runner.rb:19:in
`start'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/app_observer.rb:63:in
`react_to_state_change'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/app_observer.rb:31:in
`updated'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/app/models/runtime/app.rb:574:in
`after_commit'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/model/base.rb:1920:in
`block in _save'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:280:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:280:in
`block in remove_transaction'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:280:in
`each'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:280:in
`remove_transaction'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:156:in
`_transaction'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:108:in
`block in transaction'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/connecting.rb:250:in
`block in synchronize'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/connection_pool/threaded.rb:98:in
`hold'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/connecting.rb:250:in
`synchronize'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sequel-4.21.0/lib/sequel/database/transactions.rb:97:in
`transaction'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/app/controllers/base/model_controller.rb:66:in
`update'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/app/controllers/base/base_controller.rb:78:in
`dispatch'\",
\"/var/vcap/data/packages/cloud_controller_ng/80b067d32996057a4fc88fe1c553764671e7e8e8.1-1ea6a37e7b427cc868c0727a4bfec391725a79ab/cloud_controller_ng/lib/cloud_controller/rest_controller/routes.rb:16:in
`block in define_route'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1609:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1609:in
`block in compile!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:974:in
`[]'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:974:in
`block (3 levels) in route!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:993:in
`route_eval'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:974:in
`block (2 levels) in route!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1014:in
`block in process_route'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1012:in
`catch'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1012:in
`process_route'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:972:in
`block in route!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:971:in
`each'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:971:in
`route!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1084:in
`block in dispatch!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1066:in
`block in invoke'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1066:in
`catch'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1066:in
`invoke'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1081:in
`dispatch!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:906:in
`block in call!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1066:in
`block in invoke'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1066:in
`catch'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:1066:in
`invoke'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:906:in
`call!'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:894:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-protection-1.5.3/lib/rack/protection/xss_header.rb:18:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-protection-1.5.3/lib/rack/protection/path_traversal.rb:16:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-protection-1.5.3/lib/rack/protection/json_csrf.rb:18:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-protection-1.5.3/lib/rack/protection/base.rb:49:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-protection-1.5.3/lib/rack/protection/base.rb:49:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-protection-1.5.3/lib/rack/protection/frame_options.rb:31:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-1.6.4/lib/rack/nulllogger.rb:9:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-1.6.4/lib/rack/head.rb:13:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:181:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/sinatra-1.4.6/lib/sinatra/base.rb:2021:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-1.6.4/lib/rack/urlmap.rb:66:in
`block in call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-1.6.4/lib/rack/urlmap.rb:50:in
`each'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-1.6.4/lib/rack/urlmap.rb:50:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb:21:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/vcap_request_id.rb:14:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/cors.rb:47:in
`call_app'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/cors.rb:12:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_metrics.rb:12:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/rack-1.6.4/lib/rack/builder.rb:153:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/thin-1.6.3/lib/thin/connection.rb:86:in
`block in pre_process'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/thin-1.6.3/lib/thin/connection.rb:84:in
`catch'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/thin-1.6.3/lib/thin/connection.rb:84:in
`pre_process'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/eventmachine-1.0.8/lib/eventmachine.rb:1062:in
`call'\",
\"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/vendor/bundle/ruby/2.2.0/gems/eventmachine-1.0.8/lib/eventmachine.rb:1062:in
`block in
spawn_threadpool'\"]}","log_level":"error","source":"cc.api","data":{"request_guid":"e046d57f-277a-4f81-452c-3b6fb892432e::53557eba-6212-4727-9b43-ebbb03426b76"},"thread_id":69975079694320,"fiber_id":69975079888800,"process_id":6524,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/lib/sinatra/vcap.rb","lineno":53,"method":"block
in registered"}

{"timestamp":1449752019.691719,"message":"Completed 500 vcap-request-id:
e046d57f-277a-4f81-452c-3b6fb892432e::53557eba-6212-4727-9b43-ebbb03426b76","log_level":"info","source":"cc.api","data":{},"thread_id":69975079694320,"fiber_id":69975079888800,"process_id":6524,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":23,"method":"call"}