Changeset 926


Ignore:
Timestamp:
08/02/06 14:48:55 (7 years ago)
Author:
mglb1
Message:

Reworked template generation support

  • Templates are now generated in threads this HUGELY improves the speed eg. from 2mins -> 30s to regenerate all templates for a network of 30 hosts with 11 host templates and 31 network templates
  • This also makes template generation an asynchronous process, calling processTemplate will return a token immediately. The caller is expected to regularly poll getTemplateStatus to see how the template generation is proceeding. Generation results are stored for 30 minutes before being removed.
  • Templates should derive from the ccs_template class not ccs_host or ccs_network as previously required. All templates now receive the same data.
  • Templates in the host directory receive extra data in the top level namespace which simply consists of the entry out of the hosts array that corresponds to the host they are being processed for
Location:
ccsd
Files:
52 edited

Legend:

Unmodified
Added
Removed

  • ccsd/private/templates/host/cf_hostvars.tmpl

    r589 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cf.hostvars" 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/host/dhcpd_conf.tmpl

    r516 r926  
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    1111#attr $fileName = "dhcpd.conf" 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
    14 #def enabledOnHost($host_id): #return self.hasServiceEnabledByName("dhcp") 
     12#attr $serviceName = "dhcp" 
     13#from crcnetd._utils.ccsd_cfengine import ccs_template 
     14#extends ccs_template 
    1515#compiler-settings 
    1616## Change the Cheetah start tokens so that we don't conflict with DHCP syntax 

  • ccsd/private/templates/host/dhcpd_default.tmpl

    r516 r926  
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    1111#attr $fileName = "dhcpd_default" 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
    14 #def enabledOnHost($host_id): #return self.hasServiceEnabledByName("dhcp") 
     12#attr $serviceName = "dhcp" 
     13#from crcnetd._utils.ccsd_cfengine import ccs_template 
     14#extends ccs_template 
    1515#compiler-settings 
    1616## Change the Cheetah start tokens so that we don't conflict with DHCP syntax 

  • ccsd/private/templates/host/hosts.tmpl

    r516 r926  
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    1111 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414127.0.0.1       $host_name  localhost 
    1515 

  • ccsd/private/templates/host/interface_rules.tmpl

    r591 r926  
    1313#attr $eventList = ["hostAdded", "hostModified" ] 
    1414#attr $fileName = "interface.rules" 
    15 #from crcnetd.modules.ccs_host import ccs_host 
    1615#from crcnetd.modules.ccs_interface import INTERFACE_TYPE_VAP 
    1716#from crcnetd.modules.ccs_asset import ccs_subasset 
    18 #extends ccs_host 
     17#from crcnetd._utils.ccsd_cfengine import ccs_template 
     18#extends ccs_template 
    1919# Interface naming rules for the crcnet-bpc Distribution 
    2020# 

  • ccsd/private/templates/host/interfaces.tmpl

    r889 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    11 #from crcnetd.modules.ccs_host import ccs_host 
    1211#from crcnetd.modules.ccs_asset import ccs_subasset 
    13 #extends ccs_host 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414## 
    1515## Real template output starts below here 

  • ccsd/private/templates/host/ospfd_conf.tmpl

    r565 r926  
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    1111#attr $fileName = "ospfd.conf" 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
    14 #def enabledOnHost($host_id): #return self.hasServiceEnabledByName("quagga") 
     12#attr $serviceName = "quagga" 
     13#from crcnetd._utils.ccsd_cfengine import ccs_template 
     14#extends ccs_template 
    1515! 
    1616! CRCnet BiscuitPC Quagga Configuration 

  • ccsd/private/templates/host/pxemap.tmpl

    r540 r926  
    1212#attr $eventList = ["hostAdded", "hostModified" ] 
    1313#attr $multiFile = True 
    14 #from crcnetd.modules.ccs_host import ccs_host 
    15 #extends ccs_host 
     14#from crcnetd._utils.ccsd_cfengine import ccs_template 
     15#extends ccs_template 
    1616#for $name,$iface in $interfaces.items() 
    1717#if not $iface.interface_active 

  • ccsd/private/templates/host/quagga_daemons.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    11 #from crcnetd.modules.ccs_host import ccs_host 
    12 #extends ccs_host 
    13 #def enabledOnHost($host_id): #return self.hasServiceEnabledByName("quagga") 
     11#attr $serviceName = "quagga" 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens 

  • ccsd/private/templates/host/ssmtp_conf.tmpl

    r516 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "ssmtp.conf" 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515cheetahVarStartToken = % 

  • ccsd/private/templates/host/zebra_conf.tmpl

    r516 r926  
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    1111#attr $fileName = "zebra.conf" 
    12 #from crcnetd.modules.ccs_host import ccs_host 
    13 #extends ccs_host 
    14 #def enabledOnHost($host_id): #return self.hasServiceEnabledByName("quagga") 
     12#attr $serviceName = "quagga" 
     13#from crcnetd._utils.ccsd_cfengine import ccs_template 
     14#extends ccs_template 
    1515! 
    1616! CRCnet BiscuitPC Quagga Configuration 

  • ccsd/private/templates/network/cfconf/cf_crcnetbpc.tmpl

    r621 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cf.crcnetbpc" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/cf_dhcp.tmpl

    r614 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cf.dhcp" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/cf_groups.tmpl

    r620 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cf.groups" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/cf_main.tmpl

    r664 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cf.main" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/cf_quagga.tmpl

    r614 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cf.quagga" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/cfagent_conf.tmpl

    r516 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cfagent.conf" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/cfservd_conf.tmpl

    r516 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cfservd.conf" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfconf/update_conf.tmpl

    r516 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "update.conf" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfrun_hosts.tmpl

    r516 r926  
    1010#attr $eventList = ["hostAdded", "hostModified" ] 
    1111#attr $fileName = "cfrun.hosts" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/cfservd_conf.tmpl

    r516 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "cfservd.conf" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/crcnet-bpc/cf_defaults.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#compiler-settings 
    1414## Change the Cheetah start tokens so that we don't conflict with CFengine 

  • ccsd/private/templates/network/crcnet-bpc/crcnet_monitor.tmpl

    r618 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#compiler-settings 
    1414cheetahVarStartToken = % 

  • ccsd/private/templates/network/crcnet-bpc/crontab.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#compiler-settings 
    1414cheetahVarStartToken = % 

  • ccsd/private/templates/network/crcnet-bpc/group.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#raw 
    1414root:*:0: 

  • ccsd/private/templates/network/crcnet-bpc/network_options.tmpl

    r516 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "network_options" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515cheetahVarStartToken = % 

  • ccsd/private/templates/network/crcnet-bpc/passwd.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#raw 
    1414root:$1$vO4KGFI9$9IKT5NcQXO544HAr/u4Jw1:0:0:root:/root:/bin/bash 

  • ccsd/private/templates/network/crcnet-bpc/root_authorized_keys.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#raw 
    1414ssh-dss AAAAB3NzaC1kc3MAAACBALCyUdXLW2ZCsfMzzSYuxrvpxeljcRlxylKJ9Cvnut5twGNEhq+XusfkCqTFMRBP+qMdMZjtnqIdWWSKPrC6q1byt+ho0W6vJ5yBGiAYta0AajrHwxyb7cy1u7JCzfiSDHjmx1gciHg5NBF02HyTeIftS30XGt6dOnv7Ok1fxnSjAAAAFQCMASNUwkEBsQAj5vQWSRqsYpE6bQAAAIAiS3Pihb+DCHVeqnIDbhIQ5QyQg0zdMAHxbaW3cVoiI4F9Z7dxAT5HT6KBdzPz9rIiHTekOWdrNRqLV8SrefdzTgy2d6wBI1EnMK7V/7/w1EqFb8HZnJkOIwCB34hzGwLbeNmwjlxfBaBYuDq36q/Ojv+zn7RYgMqsEgvekoRh/wAAAIB7J9yqK7LRJDPF04ac092zswm6KW0umLY2eHY3lWaOEt6x1B27LraISj9MDd2T3xrV69N7kNs2X2q4dJao3z9FCuqspombbxpmLfQ9MpyCKtB4D9hNK7rY07b5H7YRBWo5Vutys1l/JntkCQsI/+BEArhfQs763q66+pZiaBZqwg== CRCnet router root key 

  • ccsd/private/templates/network/crcnet-bpc/sources_list.tmpl

    r615 r926  
    1010#attr $eventList = [] 
    1111#attr $fileName = "sources.list" 
    12 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    13 #extends ccs_network 
     12#from crcnetd._utils.ccsd_cfengine import ccs_template 
     13#extends ccs_template 
    1414#compiler-settings 
    1515cheetahVarStartToken = % 

  • ccsd/private/templates/network/crcnet-bpc/sshd_config.tmpl

    r516 r926  
    99#attr $license = "No usage or redistribution rights are granted. See LICENSE for details." 
    1010#attr $eventList = [] 
    11 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    12 #extends ccs_network 
     11#from crcnetd._utils.ccsd_cfengine import ccs_template 
     12#extends ccs_template 
    1313#compiler-settings 
    1414cheetahVarStartToken = % 

  • ccsd/private/templates/network/services/interface_dns_f.tmpl

    r844 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "interface_dns_f" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010cheetahVarStartToken = % 

  • ccsd/private/templates/network/services/interface_dns_r.tmpl

    r844 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "interface_dns_r" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010cheetahVarStartToken = % 

  • ccsd/private/templates/network/services/loopback_dns_f.tmpl

    r844 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "loopback_dns_f" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010cheetahVarStartToken = % 

  • ccsd/private/templates/network/services/loopback_dns_r.tmpl

    r844 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "loopback_dns_r" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010cheetahVarStartToken = % 

  • ccsd/private/templates/network/services/mrtg_cfg.tmpl

    r629 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "mrtg.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with MRTG 

  • ccsd/private/templates/network/services/nagios/checkcommands_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "checkcommands.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/contactgroups_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "contactgroups.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/contacts_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "contacts.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/dependencies_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "dependencies.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/escalations_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "escalations.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/hostextinfo_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "hostextinfo.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/hostgroups_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "hostgroups.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/hosts_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "hosts.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/local_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "local.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/misccommands_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "misccommands.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/nagios_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "nagios.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/resource_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "resource.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/services_cfg.tmpl

    r625 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "services.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/private/templates/network/services/nagios/timeperiods_cfg.tmpl

    r594 r926  
    55#attr $eventList = [] 
    66#attr $fileName = "timeperiods.cfg" 
    7 #from crcnetd._utils.ccsd_cfengine import ccs_network 
    8 #extends ccs_network 
     7#from crcnetd._utils.ccsd_cfengine import ccs_template 
     8#extends ccs_template 
    99#compiler-settings 
    1010## Change the Cheetah start tokens so that we don't conflict with Nagios 

  • ccsd/trunk/crcnetd/_utils/ccsd_cfengine.py

    r897 r926  
    3131import imp 
    3232import time 
     33import threading 
    3334from twisted.web import resource, server 
    3435from tempfile import mkdtemp 
    … …  
    4041from crcnetd._utils.ccsd_config import config_get, config_get_required 
    4142from crcnetd._utils.ccsd_session import getSession, getSessionE 
    42 from crcnetd._utils.ccsd_server import registerResource, exportViaXMLRPC 
    43  
     43from crcnetd._utils.ccsd_server import registerResource, exportViaXMLRPC, \ 
     44        initThread, registerRecurring 
     45 
     46DEFAULT_MAX_THREADS = 10 
    4447 
    4548class ccs_template_error(ccsd_error): 
    … …  
    145148    return templates 
    146149 
    147 def processEvent(eventName, host_id, session_id, **params): 
    148     """Callback process from the event module.  
    149      
    150     This function is called when an event that a template has indicated 
    151     an interest in is fired. Look through our templates to find which 
    152     one is interested and call it. 
    153     """ 
    154     global _hostTemplates, _networkTemplates 
    155  
    156     # Check session to see if a revision is active 
    157     session = getSessionE(session_id) 
    158     if session.revision == None: 
    159         return 
    160  
    161     # Fire all the templates attached to this event 
    162     for templateID in _events[eventName]: 
    163         try: 
    164             # Process host templates if applicable 
    165             if host_id != -1 and templateID in _hostTemplates.keys(): 
    166                 generateHostTemplate(session_id, host_id, templateID) 
    167             # Process any network templates 
    168             if templateID in _networkTemplates.keys(): 
    169                 generateNetworkTemplate(session_id, templateID) 
    170         except: 
    171             log_warn("Failed to process template %s!" % templateID, \ 
    172                     sys.exc_info()) 
    173             continue 
    174  
    175 @exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR) 
    176 def generateHostConfig(session_id, host_id): 
    177     """Generates configuration files for all host templates. 
    178      
    179     If host_id is -1 then the configuration is regenerated for all hosts 
    180     """ 
    181     global _hostTemplates, _networkTemplates 
    182     from crcnetd.modules.ccs_host import validateHostId, getHostList, \ 
    183             getHostName 
    184      
    185     # Get list of hosts to process 
    186     hosts = [] 
    187     if host_id != -1: 
    188         validateHostId(session_id, host_id) 
    189         hosts.append(host_id) 
    190         desc = "for %s" % getHostName(session_id, host_id) 
    191     else: 
    192         tmp = getHostList(session_id) 
    193         for host in tmp: 
    194             hosts.append(host["host_id"]) 
    195         desc = "for all hosts" 
    196  
    197     # Check session to see if a revision is active 
    198     session = getSessionE(session_id) 
    199     commit=0 
    200     if session.revision == None: 
    201         session.begin("Updated host configuration files %s" % desc, \ 
    202                 initiator="cfengine") 
    203         commit=1 
    204  
    205     # Generate the files 
    206     try: 
    207         for thost_id in hosts: 
    208             for templateID in _hostTemplates.keys(): 
    209                 generateHostTemplate(session_id, thost_id, templateID) 
    210     except: 
    211         if commit: 
    212             session.rollback() 
    213         log_error("Host Template (%s) failed!", sys.exc_info()) 
    214         (type, value, tb) = sys.exc_info() 
    215         raise ccs_cfengine_error("Host template (%s) failed: %s" % \ 
    216                 (templateID, value)) 
    217     
    218     # Update network level templates 
    219     try: 
    220         for templateID in _networkTemplates.keys(): 
    221             generateNetworkTemplate(session_id, templateID) 
    222     except: 
    223         if commit: 
    224             session.rollback() 
    225         (type, value, tb) = sys.exc_info() 
    226         log_debug("Failed to generate template!", (type, value, tb)) 
    227         raise ccs_cfengine_error("Network template (%s) failed: %s" % \ 
    228                 (templateID, value)) 
    229              
    230     # Commit if neceesary 
    231     if commit: 
    232         rv = session.commit() 
    233         return rv["revision"] 
    234      
    235     return -1 
    236  
    237 def generateNetworkTemplate(session_id, templateID): 
    238     """Creates the config for the template""" 
    239     global _networkTemplates 
    240      
    241     # Get path details 
    242     session = getSessionE(session_id) 
    243     if session.revision == None: 
    244         raise ccs_cfengine_error("Cannot generate template on session " \ 
    245                 "without revision!") 
    246     outputPath = session.revision.getConfigBase() 
    247      
    248     # Initialise the template class 
    249     template = _networkTemplates[templateID] 
    250     t = eval("template.%s(session_id)" % template.templateName) 
    251  
    252     # Write out the new template file 
    253     filename = "%s/%s" % (outputPath, template.fileName) 
    254     t.writeTemplate(filename, False) 
    255     # Set the Date keyword property on the generated file 
    256     session.revision.propset(filename, "svn:keywords", "Date") 
    257      
    258 def generateHostTemplate(session_id, host_id, templateID): 
    259     """Creates the config for the specified host and template""" 
    260     global _hostTemplates 
    261     from crcnetd.modules.ccs_host import getHostName 
    262      
    263     # Get path details 
    264     session = getSessionE(session_id) 
    265     if session.revision == None: 
    266         raise ccs_cfengine_error("Cannot generate template on session " \ 
    267                 "without revision!") 
    268     hostPath = "%s/hosts/%s" % (session.revision.getConfigBase(), \ 
    269             getHostName(session_id, host_id)) 
    270      
    271     # Initialise the template class 
    272     template = _hostTemplates[templateID] 
    273     t = eval("template.%s(session_id, host_id)" % template.templateName) 
    274     if not t.enabledOnHost(host_id): 
    275         return 
    276  
    277     # Write out the new template file 
    278     filename = "%s/%s" % (hostPath, template.fileName) 
    279     files = t.writeTemplate(filename, template.multiFile) 
    280     # Set the Date keyword property on the generated file(s) 
    281     for nfile in files: 
    282         session.revision.propset(nfile, "svn:keywords", "Date") 
    283  
    284150def registerEvents(templates): 
    285151    """Runs through the list of templates and registers callbacks""" 
    … …  
    306172                (templateID, eventName), sys.exc_info()) 
    307173 
     174def processEvent(eventName, host_id, session_id, **params): 
     175    """Callback process from the event module.  
     176     
     177    This function is called when an event that a template has indicated 
     178    an interest in is fired. Look through our templates to find which 
     179    one is interested and call it. 
     180    """ 
     181    global _hostTemplates, _networkTemplates 
     182    from crcnetd.modules.ccs_host import getHostName 
     183     
     184    # Check session to see if a revision is active 
     185    session = getSessionE(session_id) 
     186    if session.revision == None: 
     187        return 
     188 
     189    host_name = getHostName(session_id, host_id) 
     190    hosts = {host_name:[],"network":[]} 
     191    # Fire all the templates attached to this event 
     192    for templateID in _events[eventName]: 
     193        # Process host templates if applicable 
     194        if host_id != -1 and templateID in _hostTemplates.keys(): 
     195            hosts[host_name].append(templateID) 
     196        # Process any network templates 
     197        if templateID in _networkTemplates.keys(): 
     198            hosts["network"].append(templateID) 
     199 
     200    # Fire off the generation job 
     201    generateTemplate(session_id, hosts) 
     202     
     203@registerRecurring(60*10) 
     204def cleanTemplateStatus(): 
     205    """Runs every ten minutes to clean up expired template status info""" 
     206    global _templateStats, _statsLock 
     207     
     208    # If generation is not finished by this many minutes after initiation 
     209    # there is an error, stats are removed in hope of causing a Key error  
     210    # that will kill the errant threads 
     211    err_time = 60*60 
     212    # Statistics for finished generation runs are removed this many minutes 
     213    # after the run completes 
     214    expire_time = 60*30 
     215     
     216    _statsLock.acquire() 
     217    try: 
     218        for key,stats in _templateStats.items(): 
     219            if (time.time()-stats["initiated"]) > err_time: 
     220                # Still processing after one hour!? 
     221                log_error("Template generation (%s) was still active after " \ 
     222                        "%d seconds!" % (key, err_time)) 
     223                del _templateStats[key] 
     224                continue 
     225            if stats["finished"] == 0: 
     226                # Still in progress 
     227                continue 
     228            if (time.time()-stats["finished"]) > expire_time: 
     229                log_info("Removing template generation status (%s)" % key) 
     230                del _templateStats[key] 
     231    finally: 
     232        _statsLock.release() 
     233 
     234    return True 
     235     
     236@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR) 
     237def getTemplateStatus(session_id, statsKey): 
     238    """Returns the status of a specified template generation run""" 
     239    global _templateStats, _statsLock 
     240 
     241    stats = None 
     242     
     243    _statsLock.acquire() 
     244    try: 
     245        if statsKey not in _templateStats.keys(): 
     246            raise ccs_cfengine_error("Invalid key") 
     247        stats = dict(_templateStats[statsKey]) 
     248    finally: 
     249        _statsLock.release() 
     250 
     251    return stats 
     252 
     253@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR) 
     254def generateTemplate(session_id, requested_templates={}): 
     255    """Generates the specified configuration templates 
     256 
     257    The requested_templates parameter should be a dictionary indexed by  
     258    host_name each entry should contain a list of the template ids that  
     259    should be generated for that host.  
     260     
     261    If the template list for a host is empty all available templates will be  
     262    regenerated.  
     263    If the dictionary is completely empty then all hosts will have their  
     264    templates regenerated. 
     265 
     266    If an entry in the dictionary contains the key "network" it will be 
     267    interpreted as a list of network wide templates to regenerate. If this key 
     268    is absent then the network wide templates will be regenerated if the number 
     269    of host templates being regenerated is greater than one. 
     270     
     271    All template processing is shunted off to another thread and the user is 
     272    returned a token that can be passed to future calls to getTemplateStatus 
     273    to retrieve the current status of template generation. Generation results 
     274    are stored for 30 minutes after the end of template generation. 
     275    """ 
     276    global _hostTemplates, _networkTemplates, _templateStats, _statsLock 
     277    from crcnetd.modules.ccs_host import getHostList 
     278 
     279    # Setup the basic set of statistics about template generation 
     280    stats = {} 
     281    stats["planned"] = {} 
     282    stats["generated"] = {} 
     283    stats["initiated"] = time.time() 
     284    stats["generating"] = 0 
     285    stats["finished"] = 0 
     286    stats["error"] = None 
     287 
     288    # Work out what the user has requested us to generate 
     289    hosts = {} 
     290    host_names = requested_templates.keys() 
     291    if len(host_names) == 0: 
     292        host_names = [host["host_name"] for host in getHostList(session_id)] 
     293    for host in host_names: 
     294        if host == "network": continue 
     295        templates = [] 
     296        if host in requested_templates.keys(): 
     297            templates = requested_templates[host] 
     298        if len(templates) == 0: 
     299            templates = _hostTemplates.keys() 
     300        hosts[host] = templates 
     301    stats["planned"]["hosts"] = hosts 
     302    stats["generated"]["hosts"] = {} 
     303     
     304    # Work out how many host templates we're going to be generating 
     305    total = 0 
     306    for tlist in hosts.values(): 
     307        total += len(tlist) 
     308    stats["planned"]["total"] = total 
     309     
     310    # Add network level templates if more than one host template is requested 
     311    # or they have been explicitly specified 
     312    if "network" in host_names: 
     313        stats["planned"]["network"] = requested_templates["network"] 
     314    elif total > 1: 
     315        stats["planned"]["network"] = _networkTemplates.keys() 
     316    else: 
     317        stats["planned"]["network"] = [] 
     318    stats["planned"]["total"] += len(stats["planned"]["network"]) 
     319    stats["generated"]["network"] = {} 
     320         
     321    # Aquire the lock to deal with statistics 
     322    statsKey = token = createPassword(8) 
     323    _statsLock.acquire() 
     324    try: 
     325        _templateStats[statsKey] = stats 
     326    finally: 
     327        _statsLock.release() 
     328 
     329    # Fire off the new thread 
     330    initThread(processTemplates, session_id, statsKey) 
     331 
     332    # Return back to the user 
     333    return statsKey 
     334     
     335def processTemplates(session_id, statsKey): 
     336    """Does the hardwork of generating templates in a thread""" 
     337    global _templateStats, _statsLock, _threadLimit 
     338    from crcnetd.modules.ccs_host import getHostList, ccs_host, \ 
     339            getDistributions 
     340    from crcnetd._utils.ccsd_service import getServiceTemplateVars 
     341    from crcnetd.modules.ccs_asset import getAssetTypeTemplateVariables 
     342 
     343    log_info("Entered template processing thread (%s)" % statsKey) 
     344     
     345    # Make sure we have a revision to insert files into 
     346    # Check session to see if a revision is active 
     347    session = getSessionE(session_id) 
     348    commit=0 
     349    if session.revision == None: 
     350        session.begin("Updated host configuration files", initiator="cfengine") 
     351        commit=1 
     352    outputDir = session.revision.getConfigBase() 
     353     
     354    # Load all the data that we need to pass to the templates 
     355    variables = {} 
     356    hlist = {} 
     357    id2name = {} 
     358    for host in getHostList(session_id): 
     359        host = ccs_host(session_id, host["host_id"]) 
     360        hostDetails = host.getTemplateVariables() 
     361        hlist[host["host_name"]] =  hostDetails 
     362    variables["hosts"] = hlist 
     363    dlist = {} 
     364    for distrib in getDistributions(session_id): 
     365        dlist[distrib["distribution_id"]] = distrib 
     366    variables["distributions"] = dlist 
     367    variables["services"] = getServiceTemplateVars(session_id) 
     368    variables["date"] = time.ctime() 
     369    variables["domain"] = config_get_required("network", "domain") 
     370    variables["site_name"] = config_get_required("network", "site_name") 
     371    variables["server_name"] = config_get_required("network", "server_name") 
     372    variables["policy_ip"] = getIP(variables["server_name"]) 
     373    variables["smtp_server"] = config_get_required("network", "smtp_server") 
     374    variables["admin_email"] = config_get_required("network", "admin_email") 
     375   
     376    variables["asset_types"] = getAssetTypeTemplateVariables(session_id) 
     377    variables["session_id"] = session_id 
     378 
     379    # Update the stats and get the list of hosts 
     380    _statsLock.acquire() 
     381    try: 
     382        planned = _templateStats[statsKey]["planned"] 
     383        _templateStats[statsKey]["generating"] = time.time() 
     384    finally: 
     385        # Release the lock before proceeding 
     386        _statsLock.release() 
     387     
     388    # Now generate each hosts template in a separate thread 
     389    for host,template_ids in planned["hosts"].items(): 
     390        # See if we can start a new thread 
     391        _threadLimit.acquire() 
     392        try: 
     393            # Extract the host specific data for this template 
     394            hostData = variables["hosts"][host] 
     395            # Fire off the thread to process this host, the thread will release 
     396            # the semaphore as it is about to exit 
     397            # XXX: This relies on the thread behaving nicely, need to find a 
     398            # way to find out if it's been bad and exited without releasing  
     399            # the semaphore 
     400            initThread(processHostTemplates, statsKey, outputDir, hostData, \ 
     401                    variables) 
     402        except: 
     403            log_error("Failed to start thread to process templates for " \ 
     404                    "host: %s!" % host, sys.exc_info()) 
     405            _threadLimit.release() 
     406             
     407    # Wait until all the configuration files have been generated 
     408    hosts = list(planned["hosts"].keys()) 
     409    hasError = False 
     410    hasTemplateError = False 
     411    while len(hosts) > 0: 
     412        # Wait a bit before trying again 
     413        time.sleep(1) 
     414        _statsLock.acquire() 
     415        try: 
     416            generated = _templateStats[statsKey]["generated"]["hosts"] 
     417            for host in hosts: 
     418                if not host in generated.keys(): 
     419                    continue 
     420                # Check if it finished successfully 
     421                if generated[host]["finished"] > 0: 
     422                    # Remove from list 
     423                    hosts.remove(host) 
     424                    # Check if any of the templates failed to be output 
     425                    if generated[host]["error"] == "template": 
     426                        # Set the flag so we don't commit the revision 
     427                        hasTemplateError = True 
     428                else: 
     429                    # Check for fatal errors 
     430                    if generated[host]["error"] != "": 
     431                        # Set the flag and remove the host from processing 
     432                        hosts.remove(host) 
     433                        hasError = True 
     434                        continue 
     435        finally: 
     436            _statsLock.release() 
     437     
     438    # Hosts are done, network templates now 
     439    for template_id in planned["network"]: 
     440        # See if we can start a new thread 
     441        _threadLimit.acquire() 
     442        try: 
     443            # Fire off the thread to process this template, the thread will 
     444            # release the semaphore as it is about to exit 
     445            # XXX: This relies on the thread behaving nicely, need to find a 
     446            # way to find out if it's been bad and exited without releasing  
     447            # the semaphore 
     448            initThread(processNetworkTemplate, statsKey, outputDir, \ 
     449                    template_id, variables) 
     450        except: 
     451            log_error("Failed to start thread to process network template: " \ 
     452                    "%s!" % template_id, sys.exc_info()) 
     453            _threadLimit.release() 
     454 
     455    # Wait until all network templates are done 
     456    network = list(planned["network"]) 
     457    while len(network) > 0: 
     458        # Wait a bit before trying again 
     459        time.sleep(1) 
     460        _statsLock.acquire() 
     461        try: 
     462            generated = _templateStats[statsKey]["generated"]["network"] 
     463            for template in network: 
     464                if not template in generated.keys(): 
     465                    continue 
     466                # Check for errors 
     467                if generated[template]["error"] != "": 
     468                    hasTemplateError = True 
     469                # Remove from the list 
     470                network.remove(template) 
     471        finally: 
     472            _statsLock.release()         
     473     
     474    # Commit if neceesary and no template/host errors occured 
     475    rv = {"revision":None} 
     476    if commit and not (hasError or hasTemplateError): 
     477        rv = session.commit() 
     478         
     479    # All done 
     480    _statsLock.acquire() 
     481    try: 
     482        _templateStats[statsKey]["finished"] = time.time() 
     483        _templateStats[statsKey]["revision"] = rv["revision"] 
     484        d = _templateStats[statsKey]["finished"] - \ 
     485                _templateStats[statsKey]["initiated"] 
     486        if hasError: 
     487            _templateStats[statsKey]["error"] = "host" 
     488        elif hasTemplateError: 
     489            _templateStats[statsKey]["error"] = "template" 
     490    finally: 
     491        _statsLock.release() 
     492 
     493    log_info("Template generation completed in %0.3f seconds for %s" % \ 
     494            (d, statsKey)) 
     495    return True 
     496 
     497def processHostTemplates(statsKey, outputDir, hostData, networkData): 
     498    """Runs in a thread and generates the output files for the template""" 
     499    global _templateStats, _statsLock, _threadLimit, _hostTemplates 
     500    host_id = -1 
     501    host = "unknown" 
     502     
     503    # It is imperative that we release the semaphore before returning from 
     504    # this function or we could potentially starve the calling thread of  
     505    # workers and we'd end up in a deadlock situation! 
     506    try: 
     507        # Work out the host_id and host_name of this host 
     508        host_id = hostData["host_id"] 
     509        host = hostData["host_name"] 
     510        hostPath = "%s/hosts/%s" % (outputDir, host) 
     511        start = time.time() 
     512        log_debug("Started host generation thread for %s" % host) 
     513         
     514        # Retrieve the list of templates to generate and flag the start  
     515        _statsLock.acquire() 
     516        try: 
     517            planned = _templateStats[statsKey]["planned"] 
     518            generated = _templateStats[statsKey]["generated"]["hosts"] 
     519            generated[host] = {} 
     520            generated[host]["initiated"] = start 
     521            generated[host]["finished"] = 0 
     522            generated[host]["error"] = "" 
     523        finally: 
     524            _statsLock.release() 
     525 
     526        # Now that we have a set of templates to use do the actual generation 
     527        hadError = False 
     528        for tname in planned["hosts"][host]: 
     529            try: 
     530                tstart = time.time() 
     531                # Instantiate a template 
     532                template = _hostTemplates[tname] 
     533                t = eval("template.%s()" % template.templateName) 
     534                # Is it enabled on this host? 
     535                if not t.enabledOnHost(networkData["session_id"], host_id): 
     536                    continue 
     537                # Set up the variables we want to substitute in 
     538                t._searchList = [hostData, networkData] 
     539                # Generate the file 
     540                filename = "%s/%s" % (hostPath, template.fileName) 
     541                files = t.writeTemplate(filename, template.multiFile) 
     542                tend = time.time() 
     543                # Record how long the file took to generate 
     544                _statsLock.acquire() 
     545                try: 
     546                    info = {"time":tend-tstart, "error":""} 
     547                    generated[host][tname] = info 
     548                finally: 
     549                    _statsLock.release() 
     550            except: 
     551                (type, value, tb) = sys.exc_info() 
     552                # Single template errors can be skipped over, nothing will 
     553                # get committed unless the user explicitly approves 
     554                _statsLock.acquire() 
     555                try: 
     556                     
     557                    info = {"time":0, "error":value} 
     558                    generated[host][tname] = info 
     559                finally: 
     560                    _statsLock.release() 
     561                hadError = True 
     562                log_error("Failed to process template (%s) for host: %s" % \ 
     563                        (tname, host), (type, value, tb)) 
     564         
     565        # Clean up 
     566        end = time.time() 
     567        log_debug("Completed host generation for %s in %0.3f seconds" % \ 
     568                (host, (end-start))) 
     569        _statsLock.acquire() 
     570        try: 
     571            generated[host]["finished"] = end 
     572            # If a template failed make a note of it here 
     573            if hadError: 
     574                generated[host]["error"] = "template" 
     575        finally: 
     576            _statsLock.release() 
     577    except: 
     578        (type, value, tb) = sys.exc_info() 
     579        # Record the error for display to the user 
     580        _statsLock.acquire() 
     581        try: 
     582            generated = _templateStats[statsKey]["generated"]["hosts"] 
     583            if host not in generated.keys(): 
     584                generated[host] = {} 
     585                generated[host]["initiated"] = 0 
     586            generated[host]["finished"] = time.time() 
     587            generated[host]["error"] = value 
     588        finally: 
     589            _statsLock.release() 
     590        # Log the error for the administrators record 
     591        log_error("Exception while processing host templates (%s)!" % \ 
     592                host, (type, value, tb)) 
     593     
     594    # Release the semaphore 
     595    _threadLimit.release() 
     596    return 
     597 
     598def processNetworkTemplate(statsKey, outputDir, template_id, networkData): 
     599    """Runs in a thread and generates the output files for the template""" 
     600    global _templateStats, _statsLock, _threadLimit, _networkTemplates 
     601     
     602    # It is imperative that we release the semaphore before returning from 
     603    # this function or we could potentially starve the calling thread of  
     604    # workers and we'd end up in a deadlock situation! 
     605    try: 
     606        log_debug("Started template generation thread for %s" % template_id) 
     607        tstart = time.time() 
     608        # Instantiate a template 
     609        template = _networkTemplates[template_id] 
     610        t = eval("template.%s()" % template.templateName) 
     611        # Set up the variables we want to substitute in 
     612        t._searchList = [networkData] 
     613        # Generate the file 
     614        filename = "%s/%s" % (outputDir, template.fileName) 
     615        files = t.writeTemplate(filename, template.multiFile) 
     616        tend = time.time() 
     617        # Record how long the file took to generate 
     618        _statsLock.acquire() 
     619        try: 
     620            generated = _templateStats[statsKey]["generated"]["network"] 
     621            info = {"time":tend-tstart, "error":""} 
     622            generated[template_id] = info 
     623        finally: 
     624            _statsLock.release() 
     625        log_debug("Completed template generation for %s in %0.3f seconds" \ 
     626                % (template_id, (tend-tstart))) 
     627    except: 
     628        (type, value, tb) = sys.exc_info() 
     629        # Skip over the error, nothing will get committed unless the user 
     630        # explicitly approves 
     631        _statsLock.acquire() 
     632        try: 
     633            generated = _templateStats[statsKey]["generated"]["network"] 
     634            info = {"time":0, "error":value} 
     635            generated[template_id] = info 
     636        finally: 
     637            _statsLock.release() 
     638        log_error("Failed to process network template (%s)" % template_id, \ 
     639                (type, value, tb)) 
     640     
     641    # Release the semaphore 
     642    _threadLimit.release() 
     643    return 
     644 
    308645##################################################################### 
    309646# Template Mixin 
    … …  
    312649    """Config System Template Processor 
    313650     
    314     This class acts as a Mix-In to extends the base Cheetah Template class 
    315     and add template generation functionality to any class that is derived 
    316     from ccs_class.  
    317  
    318     A class that wants to make itself available to the template system should 
    319     be declared as: 
    320     class ccs_foo(ccs_class, ccs_template) 
    321  
    322     If the class redeclares __init__ then it *must* ensure that it also calls 
    323     the superclasses init (ie. ccs_template.__init__(self) ). 
    324  
    325     To generate the template output simply treat the derived class as string, 
    326     or call the fillTemplate or writeTemplate methods.  
    327  
    328     The placeholders available to the template will be sourced from the  
    329     getTemplateVariables function of the class.  
     651    All templates that expect to be processed by the configuration system 
     652    must derive from this class. It provides helper functions that are required 
     653    to determine where and when each template should be generated and how to 
     654    store it to disk.  
    330655    """ 
    331656 
    … …  
    338663        Template.__init__(self) 
    339664 
    340     def fillTemplate(self): 
    341         """Returns the processed template 
    342          
    343         Assigns variables from the class to the template and then causes the 
    344         template to run, performing all substitutions.  
    345         """ 
    346         from crcnetd.modules.ccs_asset import getAssetTypeTemplateVariables 
    347  
    348         # Build a dictionary of common variables that should be available 
    349         # to all templates 
    350         vars = {} 
    351         vars["date"] = time.ctime() 
    352         vars["domain"] = self.domain 
    353         vars["site_name"] = self.site_name 
    354         vars["server_name"] = self.server_name 
    355         vars["policy_ip"] = getIP(self.server_name) 
    356         vars["smtp_server"] = self.smtp_server 
    357         vars["admin_email"] = self.admin_email 
    358          
    359         vars["asset_types"] = getAssetTypeTemplateVariables(self._session_id) 
    360         vars["session_id"] = self._session_id 
    361          
    362         self._searchList = [self.getTemplateVariables(), vars] 
    363  
    364         return self.writeBody() 
    365  
    366665    def writeTemplate(self, filename, multiFile): 
    367666        """Writes the template output to the specified file""" 
    … …  
    373672 
    374673        # Get the template contents 
    375         template = self.fillTemplate().strip() 
    376         if not template.endswith("\n"): template = "%s\n" % template 
     674        template = self.writeBody().strip() 
     675        if not template.endswith("\n"): template += "\n" 
    377676 
    378677        # Write it out to a file 
    … …  
    407706 
    408707    def __str__(self): 
    409         return self.fillTemplate() 
    410      
    411     def enabledOnHost(self, host_id): 
     708        return self.writeBody() 
     709     
     710    def enabledOnHost(self, session_id, host_id): 
    412711        """Returns true if the template is applicable to the specified host 
    413712 
    414         Typically this method is used by service templates to disable 
    415         processing of a template when the service is not enabled on the 
    416         specified host. 
    417          
    418         This function returns true by default, you must override it in your 
    419         implementing class/template if you want to implement more logic. 
     713        By default this method looks to see if the template has defined a  
     714        serviceName parameter. If it has then the function checks to see if 
     715        that service is enabled on the specified host. If it is not the  
     716        function returns False.  
     717         
     718        This function may be overriden by other classes/templates if you want 
     719        to implement more logic than the default implementation provides. 
    420720        """ 
    421         return True 
     721        serviceName = getattr(self, "serviceName", None) 
     722        if serviceName is None: 
     723            return True 
     724 
     725        from crcnetd.modules.ccs_host import ccs_host 
     726        host = ccs_host(session_id, host_id) 
     727        return host.hasServiceEnabledByName(serviceName) 
    422728 
    423729    def getTemplateVariables(self): 
    … …  
    431737        """ 
    432738        return [] 
    433  
    434 ##################################################################### 
    435 # Helper Class for network tempaltes to get ALL data 
    436 ##################################################################### 
    437 class ccs_network(ccs_class, ccs_template): 
    438      
    439     def __init__(self, session_id): 
    440         """Initialises a new class for the network. 
    441  
    442         The specified session must be valid and have appropriate access to 
    443         the database for the tasks you intend to perform with the class. All 
    444         database access / configuration manipulation triggered by this 
    445         instance will pass through the specified session. 
    446         """ 
    447  
    448         self._errMsg = "" 
    449         self._commit = 0 
    450         self._csInit = "" 
    451          
    452         session = getSession(session_id) 
    453         if session is None: 
    454             raise ccs_cfengine_error("Invalid session id") 
    455         self._session_id = session_id 
    456          
    457         # Store details  
    458         self._properties = {} 
    459  
    460         # Now call the base class 
    461         ccs_template.__init__(self) 
    462          
    463     def getTemplateVariables(self): 
    464         """Returns as much information about the network as possible""" 
    465         from crcnetd.modules.ccs_host import ccs_host, getHostList, \ 
    466                 getDistributions 
    467         from crcnetd._utils.ccsd_service import getServiceTemplateVars 
    468  
    469         variables = {} 
    470  
    471         # Get a list of all hosts 
    472         hlist = {} 
    473         for host in getHostList(self._session_id): 
    474             host = ccs_host(self._session_id, host["host_id"]) 
    475             hostDetails = host.getTemplateVariables() 
    476             hlist[host["host_name"]] =  hostDetails 
    477         variables["hosts"] = hlist 
    478          
    479         # Get a list of distributions 
    480         dlist = {} 
    481         for distrib in getDistributions(self._session_id): 
    482             dlist[distrib["distribution_id"]] = distrib 
    483         variables["distributions"] = dlist 
    484  
    485         # Get a list of services 
    486         variables["services"] = getServiceTemplateVars(self._session_id) 
    487  
    488         return variables 
    489739 
    490740##################################################################### 
    … …  
    12181468_cfInputDir = "" 
    12191469_svnroot = "" 
    1220   
     1470_templateStats = {} 
     1471_statsLock = None 
     1472_threadLimit = None 
     1473 
    12211474def initCFengine(): 
    12221475    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir 
    1223     global _events, _cfBaseDir, _cfInputDir, _svnroot 
     1476    global _events, _cfBaseDir, _cfInputDir, _svnroot, _statsLock, _threadLimit 
    12241477     
    12251478    # Retrieve template directory configuration 
    … …  
    12531506    ccs_revision.svnroot = _svnroot 
    12541507     
    1255     # Setup template variables from configuration file 
    1256     ccs_template.domain = config_get_required("network", "domain") 
    1257     ccs_template.site_name = config_get_required("network", "site_name") 
    1258     ccs_template.server_name = config_get_required("network", "server_name") 
    1259     ccs_template.smtp_server = config_get_required("network", "smtp_server") 
    1260     ccs_template.admin_email = config_get_required("network", "admin_email") 
     1508    # Initialise the statistics lock 
     1509    _statsLock = threading.Lock() 
     1510 
     1511    # How many threads are we allowed to run at a time 
     1512    maxThreads = config_get("cfengine", "max_threads", DEFAULT_MAX_THREADS) 
     1513    _threadLimit = threading.Semaphore(maxThreads) 
    12611514     
    12621515    # Initialise the templates 

  • ccsd/trunk/crcnetd/_utils/ccsd_server.py

    r921 r926  
    9797     
    9898    return decorator 
     99 
     100def initThread(func, *args, **kwargs): 
     101    """Calls the specified function in a new thread""" 
     102    reactor.callInThread(func, *args, **kwargs) 
    99103 
    100104def stopCCSDServer(): 

  • ccsd/trunk/crcnetd/modules/ccs_host.py

    r626 r926  
    2828from crcnetd._utils.ccsd_session import getSession, getSessionE 
    2929from crcnetd._utils.ccsd_server import exportViaXMLRPC, registerResource 
    30 from crcnetd._utils.ccsd_cfengine import ccs_template, getCfengineHostStatus 
     30from crcnetd._utils.ccsd_cfengine import getCfengineHostStatus 
    3131from crcnetd._utils.ccsd_service import getServiceInstance, getServiceID, \ 
    3232        ccsd_service 
    … …  
    4141    pass 
    4242 
    43 class ccs_host(ccs_class, ccs_template): 
     43class ccs_host(ccs_class): 
    4444    """Provides an abstract interface to a host in the CRCnet Configuration 
    4545    System. 
Note: See TracChangeset for help on using the changeset viewer.