Re: Strange events leading to "Hotplug scripts not working" and xen death

[Avleen Vig <lists-netbsd%silverwraith.com@localhost>]

On Tue, May 15, 2007 at 07:34:19PM +0200, Manuel Bouyer wrote:
> > *Finally* :-) commands like 'xenstore-ls' just hang too.
> > The xenstore commands seem to be hanging trying to read from the store.
> 
> So it's most probably an issue with the store. Could you try ktrace to
> see on what xenstore-ls hangs ? Is xenstored still running ?
> You could try to ktrace it too.

I ran xenstore-ls while trying to create one of these broken domains.
The /local/3/device/vif/0/backend line is.. interesting:
  "backend = "/locaû""

Hopefully this problem is simply that the store is being writen to
incorrectly.
I then deleted all of the xen related files in /var/run, rebooted and
ran the `xm create` and `xenstore-ls`. This time it spat back a "cannot
allocate memory" error :-( The outputs from both runs are below.


========================================================================
First run:

[root@xen1] ~ # xenstore-ls
tool = ""       
 xenstored = "" 
vm = ""         
 00000000-0000-0000-0000-000000000000 = ""
  on_xend_stop = "ignore"
  shadow_memory = "0"
  uuid = "00000000-0000-0000-0000-000000000000"
  on_reboot = "restart"
  on_poweroff = "destroy"
  on_xend_start = "ignore"
  on_crash = "restart"
  xend = ""     
   restart_count = "0"
  vcpus = "1"
  vcpu_avail = "1"
  memory = "64"
  maxmem = "64"
  name = "Domain-0"
 00ada6e1-ff9c-facb-80b5-2acad2e97eac = ""
  image = "(linux (kernel /xen/kernels/vmlinuz-2.6.16.33-xen) (args 
'ip=10.0.0.13:10.0.0.12:10.0.0..."
   ostype = "linux"
   kernel = "/xen/kernels/vmlinuz-2.6.16.33-xen"
   cmdline = "ip=10.0.0.13:10.0.0.12:10.0.0.1:255.255.255.0::eth0:off 
root=/dev/nfs nfsroot=10.0.0..."
   ramdisk = ""
  on_xend_stop = "ignore"
  shadow_memory = "0"
  uuid = "00ada6e1-ff9c-facb-80b5-2acad2e97eac"
  on_reboot = "restart"
  start_time = "1179406408.47"
  on_poweroff = "destroy"
  on_xend_start = "ignore"
  on_crash = "restart"
  xend = ""
   restart_count = "0"
  vcpus = "1"
  vcpu_avail = "1"
  memory = "64"
  maxmem = "64"
  name = "test_os"
local = ""
 domain = ""
  0 = ""
   vm = "/vm/00000000-0000-0000-0000-000000000000"
   cpu = ""
    0 = ""
     availability = "online"
   memory = ""
    target = "65536"
   name = "Domain-0"
   console = ""
    limit = "1048576"
   domid = "0"
   backend = ""
    vif = ""
     1 = ""
     2 = ""
     3 = ""
      0 = ""
       domain = "test_os"
       handle = "0"
       uuid = "d51098d5-d658-0e05-e520-756e2f8c6a36"
       script = "/usr/pkg/etc/xen/scripts/vif-bridge"
       ip = "10.0.0.13"
       state = "2"
       frontend = "/local/domain/3/device/vif/0"
       mac = "00:16:3e:29:67:16"
       online = "1"
       frontend-id = "3"
  3 = ""
   vm = "/vm/00ada6e1-ff9c-facb-80b5-2acad2e97eac"
   device-misc = ""
    vif = ""
     nextDeviceID = "1"
   device = ""
    vif = ""
     0 = ""
      backend-id = "0"
      mac = "00:16:3e:29:67:16"
      handle = "0"
      state = "1"
      backend = "/locaû"
   console = "" 
    ring-ref = "509853"
    port = "2"
    limit = "1048576"
    tty = "/dev/ttyp2"
   name = "test_os"
   domid = "3"
   cpu = ""
    0 = ""
     availability = "online"
   memory = ""
    target = "65536"
   store = ""
    ring-ref = "509854"
    port = "1"

========================================================================
Second run:

[root@xen1] ~ # xenstore-ls
tool = ""
 xenstored = ""
vm = ""
 00000000-0000-0000-0000-000000000000 = ""
  on_xend_stop = "ignore"
  shadow_memory = "0"
  uuid = "00000000-0000-0000-0000-000000000000"
  on_reboot = "restart"
  on_poweroff = "destroy"
  on_xend_start = "ignore"
  on_crash = "restart"
  xend = ""
   restart_count = "0"
  vcpus = "1"
  vcpu_avail = "1"
  memory = "64"
  maxmem = "64"
  name = "Domain-0"
 ef25b12c-a2ed-cb7f-6bae-3d7a7f6b5965 = ""
  image = "(linux (kernel /xen/kernels/vmlinuz-2.6.16.33-xen) (args 
'ip=10.0.0.13:10.0.0.12:10.0.0.1:255.255...."
   ostype = "linux"
   kernel = "/xen/kernels/vmlinuz-2.6.16.33-xen"
   cmdline = "ip=10.0.0.13:10.0.0.12:10.0.0.1:255.255.255.0::eth0:off 
root=/dev/nfs nfsroot=10.0.0.12:/tmp/xe..."
   ramdisk = ""
  on_xend_stop = "ignore"
  shadow_memory = "0"
  uuid = "ef25b12c-a2ed-cb7f-6bae-3d7a7f6b5965"
  on_reboot = "restart"
  start_time = "1179406927.98"
  on_poweroff = "destroy"
  on_xend_start = "ignore"
  on_crash = "restart"
  xend = ""
   restart_count = "0"
  vcpus = "1"
  vcpu_avail = "1"
  memory = "64"
  maxmem = "64"
  name = "test_os"
local = ""
 domain = ""
  0 = ""
   vm = "/vm/00000000-0000-0000-0000-000000000000"
   cpu = ""
    0 = ""
     availability = "online"
   memory = ""
    target = "65536"
   name = "Domain-0"
   console = ""
    limit = "1048576"
   domid = "0"
   backend = ""
    vif = ""
     1 = ""
      0 = ""
       domain = "test_os"
       handle = "0"
       uuid = "ac21d8c9-9aba-07ef-6cb3-d2481673ce62"
       script = "/usr/pkg/etc/xen/scripts/vif-bridge"
       ip = "10.0.0.13"
       state = "2"
       frontend = "/local/domain/1/device/vif/0"
       mac = "00:16:3e:6a:69:2e"
       online = "1"
       frontend-id = "1"
  1 = ""
   vm = "/vm/ef25b12c-a2ed-cb7f-6bae-3d7a7f6b5965"
   device-misc = ""
    vif = ""
     nextDeviceID = "1"
   device = ""
    vif = ""
     0 = ""
      backend-id = "0"
      mac = "00:16:3e:6a:69:2e"
      handle = "0"
      state = "1"
      backend = "/local/domain/0/backend/vif/1/0"
   console = ""
xenstore-ls: xs_directory (/local/domain/1/console): Cannot allocate memory