source: ccsd/trunk/crcnetd/_utils/ccsd_cfengine.py @ 897

Last change on this file since 897 was 897, checked in by mglb1, 7 years ago

Modified getCfengineHostStatus to check directly in the repository for
key rather than doing a checkout. This provides a 1-2 second speedup
  • Property svn:keywords set to Id
File size: 43.8 KB
Line 
1# Copyright (C) 2006  The University of Waikato
2#
3# This file is part of crcnetd - CRCnet Configuration System Daemon
4#
5# CFengine Configuration Setup
6#
7# Manages the cfengine configuration for the configuration system. The two
8# primary tasks involved in this are:
9# - Configuration file generation from templates
10# - Controling cfengine runs and collecting output on demand
11#
12# Author:       Matt Brown <matt@crc.net.nz>
13# Version:      $Id$
14#
15# crcnetd is free software; you can redistribute it and/or modify it under the
16# terms of the GNU General Public License version 2 as published by the Free
17# Software Foundation.
18#
19# crcnetd is distributed in the hope that it will be useful, but WITHOUT ANY
20# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
22# details.
23#
24# You should have received a copy of the GNU General Public License along with
25# crcnetd; if not, write to the Free Software Foundation, Inc.,
26# 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
27import sys
28import os
29import os.path
30import pysvn
31import imp
32import time
33from twisted.web import resource, server
34from tempfile import mkdtemp
35from Cheetah.Template import Template
36
37from crcnetd._utils.ccsd_common import *
38from crcnetd._utils.ccsd_log import *
39from crcnetd._utils.ccsd_events import *
40from crcnetd._utils.ccsd_config import config_get, config_get_required
41from crcnetd._utils.ccsd_session import getSession, getSessionE
42from crcnetd._utils.ccsd_server import registerResource, exportViaXMLRPC
43
44
45class ccs_template_error(ccsd_error):
46    pass
47class ccs_cfengine_error(ccsd_error):
48    pass
49
50#####################################################################
51# General Cfengine Integration Helper Functions
52#####################################################################
53def initTemplates():
54    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
55   
56    # Ensure output template module directories exist
57    ensureDirExists("%s/host" % _templateModDir)
58    ensureDirExists("%s/network" % _templateModDir)
59
60    # Ensure templates are compiled
61    rv = os.system("/usr/bin/cheetah compile -R --idir %s/host --odir " \
62            "%s/host --nobackup &>/dev/null" % (_templateDir, _templateModDir))
63    if rv != 0:
64        log_error("Unable to compile all host templates!")
65    rv = os.system("/usr/bin/cheetah compile -R --idir %s/network --odir " \
66            "%s/network --nobackup &>/dev/null" % \
67            (_templateDir, _templateModDir))
68    if rv != 0:
69        log_error("Unable to compile all network templates!")
70   
71    # Load template files
72    _hostTemplates = loadTemplates("%s/host" % _templateModDir)
73    log_info("Loaded %s host templates for cfengine." % len(_hostTemplates))
74    _networkTemplates = loadTemplates("%s/network" % _templateModDir)
75    log_info("Loaded %s network templates for cfengine." % \
76            len(_networkTemplates))
77
78    # Register for events
79    registerEvents(_hostTemplates)
80    registerEvents(_networkTemplates)
81
82def loadTemplates(moduleDir, baseStrip=""):
83    """Creates a dictionary of template modules from the specified dir"""
84   
85    # Dictionary of template objects
86    templates = {}
87   
88    if baseStrip == "":
89        baseStrip = moduleDir
90       
91    # Get a list of possible templates
92    if os.access(moduleDir, os.R_OK) != 1:
93        raise ccs_template_exception("Unable to access directory! - %s" % \
94                moduleDir)
95    ptempls = os.listdir(moduleDir)
96   
97    # Scan through the list and load valid modules
98    for tfile in ptempls:
99        # Ignore hidden files
100        if tfile.startswith("."):
101            continue
102        tfilename = "%s/%s" % (moduleDir, tfile)
103        # Recurse into directories
104        if os.path.isdir(tfilename):
105            templates.update(loadTemplates(tfilename, baseStrip))
106            continue
107        # Ignore non module files
108        if not tfile.endswith(".py"):
109            continue
110        # Load the template module
111        tname = os.path.basename(tfilename)[:-3]
112        if tname == "__init__":
113            # Ignore python framework stuff
114            continue
115        m = None
116        try:
117            m = imp.load_source(tname, tfilename)
118            # Don't want the module in the system module list
119            if tname in sys.modules.keys():
120                del sys.modules[tname]
121        except:
122            log_debug("Module import failed for %s template" % tname, \
123                    sys.exc_info())
124            pass
125        if not m:
126            log_error("Failed to import template: %s" % tname)
127            continue
128        # If an explicit output filename was specified use that, otherwise
129        # prepend path information and store the template filename.
130        path = "%s/" % os.path.dirname(tfilename[len(baseStrip)+1:])
131        if len(path) == 1:
132            path = ""
133        tclass = eval("m.%s" % tname)
134        m.fileName = "%s%s" % (path, getattr(tclass, "fileName", tname))
135        m.multiFile = getattr(tclass, "multiFile", False)
136        m.templateName = tname
137        # Store the template for future use, prepend path for uniqueness
138        templateID = "%s%s" % (path, tname)
139        if templateID in templates.keys():
140            log_error("Could not import duplicate template: %s" % tname)
141            continue
142        templates[templateID] = m
143   
144    # Return the templates
145    return templates
146
147def processEvent(eventName, host_id, session_id, **params):
148    """Callback process from the event module.
149   
150    This function is called when an event that a template has indicated
151    an interest in is fired. Look through our templates to find which
152    one is interested and call it.
153    """
154    global _hostTemplates, _networkTemplates
155
156    # Check session to see if a revision is active
157    session = getSessionE(session_id)
158    if session.revision == None:
159        return
160
161    # Fire all the templates attached to this event
162    for templateID in _events[eventName]:
163        try:
164            # Process host templates if applicable
165            if host_id != -1 and templateID in _hostTemplates.keys():
166                generateHostTemplate(session_id, host_id, templateID)
167            # Process any network templates
168            if templateID in _networkTemplates.keys():
169                generateNetworkTemplate(session_id, templateID)
170        except:
171            log_warn("Failed to process template %s!" % templateID, \
172                    sys.exc_info())
173            continue
174
175@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
176def generateHostConfig(session_id, host_id):
177    """Generates configuration files for all host templates.
178   
179    If host_id is -1 then the configuration is regenerated for all hosts
180    """
181    global _hostTemplates, _networkTemplates
182    from crcnetd.modules.ccs_host import validateHostId, getHostList, \
183            getHostName
184   
185    # Get list of hosts to process
186    hosts = []
187    if host_id != -1:
188        validateHostId(session_id, host_id)
189        hosts.append(host_id)
190        desc = "for %s" % getHostName(session_id, host_id)
191    else:
192        tmp = getHostList(session_id)
193        for host in tmp:
194            hosts.append(host["host_id"])
195        desc = "for all hosts"
196
197    # Check session to see if a revision is active
198    session = getSessionE(session_id)
199    commit=0
200    if session.revision == None:
201        session.begin("Updated host configuration files %s" % desc, \
202                initiator="cfengine")
203        commit=1
204
205    # Generate the files
206    try:
207        for thost_id in hosts:
208            for templateID in _hostTemplates.keys():
209                generateHostTemplate(session_id, thost_id, templateID)
210    except:
211        if commit:
212            session.rollback()
213        log_error("Host Template (%s) failed!", sys.exc_info())
214        (type, value, tb) = sys.exc_info()
215        raise ccs_cfengine_error("Host template (%s) failed: %s" % \
216                (templateID, value))
217   
218    # Update network level templates
219    try:
220        for templateID in _networkTemplates.keys():
221            generateNetworkTemplate(session_id, templateID)
222    except:
223        if commit:
224            session.rollback()
225        (type, value, tb) = sys.exc_info()
226        log_debug("Failed to generate template!", (type, value, tb))
227        raise ccs_cfengine_error("Network template (%s) failed: %s" % \
228                (templateID, value))
229           
230    # Commit if neceesary
231    if commit:
232        rv = session.commit()
233        return rv["revision"]
234   
235    return -1
236
237def generateNetworkTemplate(session_id, templateID):
238    """Creates the config for the template"""
239    global _networkTemplates
240   
241    # Get path details
242    session = getSessionE(session_id)
243    if session.revision == None:
244        raise ccs_cfengine_error("Cannot generate template on session " \
245                "without revision!")
246    outputPath = session.revision.getConfigBase()
247   
248    # Initialise the template class
249    template = _networkTemplates[templateID]
250    t = eval("template.%s(session_id)" % template.templateName)
251
252    # Write out the new template file
253    filename = "%s/%s" % (outputPath, template.fileName)
254    t.writeTemplate(filename, False)
255    # Set the Date keyword property on the generated file
256    session.revision.propset(filename, "svn:keywords", "Date")
257   
258def generateHostTemplate(session_id, host_id, templateID):
259    """Creates the config for the specified host and template"""
260    global _hostTemplates
261    from crcnetd.modules.ccs_host import getHostName
262   
263    # Get path details
264    session = getSessionE(session_id)
265    if session.revision == None:
266        raise ccs_cfengine_error("Cannot generate template on session " \
267                "without revision!")
268    hostPath = "%s/hosts/%s" % (session.revision.getConfigBase(), \
269            getHostName(session_id, host_id))
270   
271    # Initialise the template class
272    template = _hostTemplates[templateID]
273    t = eval("template.%s(session_id, host_id)" % template.templateName)
274    if not t.enabledOnHost(host_id):
275        return
276
277    # Write out the new template file
278    filename = "%s/%s" % (hostPath, template.fileName)
279    files = t.writeTemplate(filename, template.multiFile)
280    # Set the Date keyword property on the generated file(s)
281    for nfile in files:
282        session.revision.propset(nfile, "svn:keywords", "Date")
283
284def registerEvents(templates):
285    """Runs through the list of templates and registers callbacks"""
286   
287    for templateID, template in templates.items():
288        events = eval("template.%s.eventList" % template.templateName)
289        if events == [] or not events:
290            continue
291        for event in events:
292            registerCallback(templateID, event)
293   
294def registerCallback(templateID, eventName):
295    """Keeps our internal list of what templates want to be called"""
296
297    if eventName in _events.keys():
298        _events[eventName].append(templateID)
299        return
300
301    try:
302        catchEvent(eventName, processEvent)
303        _events[eventName] = [templateID]
304    except ccs_event_error:
305        log_warn("Template %s not bound to event %s!" % \
306                (templateID, eventName), sys.exc_info())
307
308#####################################################################
309# Template Mixin
310#####################################################################
311class ccs_template(Template):
312    """Config System Template Processor
313   
314    This class acts as a Mix-In to extends the base Cheetah Template class
315    and add template generation functionality to any class that is derived
316    from ccs_class.
317
318    A class that wants to make itself available to the template system should
319    be declared as:
320    class ccs_foo(ccs_class, ccs_template)
321
322    If the class redeclares __init__ then it *must* ensure that it also calls
323    the superclasses init (ie. ccs_template.__init__(self) ).
324
325    To generate the template output simply treat the derived class as string,
326    or call the fillTemplate or writeTemplate methods.
327
328    The placeholders available to the template will be sourced from the
329    getTemplateVariables function of the class.
330    """
331
332    def __init__(self):
333        """Initialise the class
334
335        You must call this method from your subclass
336        """
337        # Call Cheetah's init
338        Template.__init__(self)
339
340    def fillTemplate(self):
341        """Returns the processed template
342       
343        Assigns variables from the class to the template and then causes the
344        template to run, performing all substitutions.
345        """
346        from crcnetd.modules.ccs_asset import getAssetTypeTemplateVariables
347
348        # Build a dictionary of common variables that should be available
349        # to all templates
350        vars = {}
351        vars["date"] = time.ctime()
352        vars["domain"] = self.domain
353        vars["site_name"] = self.site_name
354        vars["server_name"] = self.server_name
355        vars["policy_ip"] = getIP(self.server_name)
356        vars["smtp_server"] = self.smtp_server
357        vars["admin_email"] = self.admin_email
358       
359        vars["asset_types"] = getAssetTypeTemplateVariables(self._session_id)
360        vars["session_id"] = self._session_id
361       
362        self._searchList = [self.getTemplateVariables(), vars]
363
364        return self.writeBody()
365
366    def writeTemplate(self, filename, multiFile):
367        """Writes the template output to the specified file"""
368       
369        files = []
370       
371        # Ensure the output directory exists
372        ensureDirExists(os.path.dirname(filename))
373
374        # Get the template contents
375        template = self.fillTemplate().strip()
376        if not template.endswith("\n"): template = "%s\n" % template
377
378        # Write it out to a file
379        if not multiFile:
380            f = open(filename, "w")
381            f.write(template)
382            f.close()
383            files.append(filename)
384        else:
385            # Handle templates that generate multiple output files
386            lines = template.split("\n")
387            f = None
388            for line in lines:
389                if line.startswith(".newfile"):
390                    # New file starting, close previous file
391                    if f is not None: f.close()
392                    # Open new file
393                    parts = line.split(" ")
394                    if len(parts) != 2:
395                        raise ccs_template_error("Invalid multifile template!")
396                    fname = "%s%s" % (filename, parts[1])
397                    f = open(fname, "w")
398                    files.append(fname)
399                    continue
400                elif f is not None:
401                    # Write the line out
402                    f.write("%s\n" % line)
403            # Close the file
404            if f is not None: f.close() 
405
406        return files
407
408    def __str__(self):
409        return self.fillTemplate()
410   
411    def enabledOnHost(self, host_id):
412        """Returns true if the template is applicable to the specified host
413
414        Typically this method is used by service templates to disable
415        processing of a template when the service is not enabled on the
416        specified host.
417       
418        This function returns true by default, you must override it in your
419        implementing class/template if you want to implement more logic.
420        """
421        return True
422
423    def getTemplateVariables(self):
424        """Returns a dictionary of variables that can be used by the template
425
426        The dictionary is passed to Cheetah's searchList so that it's entries
427        can be used as placeholders in the template.
428
429        This function retains an empty list. You should override this in your
430        implementing class.
431        """
432        return []
433
434#####################################################################
435# Helper Class for network tempaltes to get ALL data
436#####################################################################
437class ccs_network(ccs_class, ccs_template):
438   
439    def __init__(self, session_id):
440        """Initialises a new class for the network.
441
442        The specified session must be valid and have appropriate access to
443        the database for the tasks you intend to perform with the class. All
444        database access / configuration manipulation triggered by this
445        instance will pass through the specified session.
446        """
447
448        self._errMsg = ""
449        self._commit = 0
450        self._csInit = ""
451       
452        session = getSession(session_id)
453        if session is None:
454            raise ccs_cfengine_error("Invalid session id")
455        self._session_id = session_id
456       
457        # Store details
458        self._properties = {}
459
460        # Now call the base class
461        ccs_template.__init__(self)
462       
463    def getTemplateVariables(self):
464        """Returns as much information about the network as possible"""
465        from crcnetd.modules.ccs_host import ccs_host, getHostList, \
466                getDistributions
467        from crcnetd._utils.ccsd_service import getServiceTemplateVars
468
469        variables = {}
470
471        # Get a list of all hosts
472        hlist = {}
473        for host in getHostList(self._session_id):
474            host = ccs_host(self._session_id, host["host_id"])
475            hostDetails = host.getTemplateVariables()
476            hlist[host["host_name"]] =  hostDetails
477        variables["hosts"] = hlist
478       
479        # Get a list of distributions
480        dlist = {}
481        for distrib in getDistributions(self._session_id):
482            dlist[distrib["distribution_id"]] = distrib
483        variables["distributions"] = dlist
484
485        # Get a list of services
486        variables["services"] = getServiceTemplateVars(self._session_id)
487
488        return variables
489
490#####################################################################
491# Functions to maintain / deal with hosts
492#####################################################################
493def getCfengineHostStatus(session_id, host):
494    """Returns a dictionary describing the state of the host configuration"""
495
496    status = {}
497   
498    # Check keys exist
499    revision = ccs_revision(checkout=False)
500   
501    sshkeydir = "inputs/sshkeys/%s" % host["host_name"]
502    cfkeydir = "ppkeys"
503   
504    # SSH keys
505    status["ssh_keys"] = {"status":STATUS_OK, "msg":[]}
506    for key in ["dsa", "rsa"]:
507        if not revision.fileExists("%s/ssh_host_%s_key" % (sshkeydir, key)):
508            status["ssh_keys"]["status"] = STATUS_CRITICAL
509            status["ssh_keys"]["msg"].append("%s private key missing" % \
510                    key.upper())
511        if not revision.fileExists("%s/ssh_host_%s_key.pub" % \
512                (sshkeydir, key)):
513            status["ssh_keys"]["status"] = STATUS_CRITICAL
514            status["ssh_keys"]["msg"].append("%s public key missing" % \
515                    key.upper())
516       
517    # Cfengine keys
518    status["cfengine_keys"] = {"status":STATUS_OK, "msg":[]}
519    if not revision.fileExists("%s/root-%s.priv" % \
520            (cfkeydir, host["ip_address"])):
521        status["cfengine_keys"]["status"] = STATUS_CRITICAL
522        status["cfengine_keys"]["msg"].append("CFengine private key missing")
523    if not revision.fileExists("%s/root-%s.pub" % \
524            (cfkeydir, host["ip_address"])):
525        status["cfengine_keys"]["status"] = STATUS_CRITICAL
526        status["cfengine_keys"]["msg"].append("CFengine public key missing")
527   
528    return status
529   
530@catchEvent("hostAdded")
531def hostAddedCB(eventName, host_id, session_id, **params):
532    """Callback function to setup host configuration upon host creation"""
533
534    session = getSession(session_id)
535    if session is None:
536        return
537
538    # Create keys for the host
539    createSSHHostKeys(session_id, host_id)
540    createCfengineHostKeys(session_id, host_id)
541
542@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
543def createSSHHostKeys(session_id, host_id):
544    """Creates a set of SSH keys for the specified host"""
545    from crcnetd.modules.ccs_host import ccs_host
546    session = getSessionE(session_id)
547   
548    host = ccs_host(session_id, host_id)
549   
550    # Check for existing changeset
551    commit = 0
552    if session.changeset == 0:
553        session.begin("Created SSH keys for %s" % host["host_name"])
554        commit = 1
555   
556    try:
557        # Determine directories to store things in
558        revdir = session.revision.getConfigBase()
559        sshkeydir = "%s/sshkeys/%s" % (revdir, host["host_name"])
560        ensureDirExists(sshkeydir)
561       
562        # Generate the keys
563        genSSHKey(sshkeydir, "rsa")
564        genSSHKey(sshkeydir, "dsa")   
565    except:
566        (type, value, tb) = sys.exc_info()
567        log_error("Failed to generate SSH keys for %s! - %s" % \
568                (host["host_name"], value))
569        # Rollback any changes
570        if commit == 1:
571            session.rollback()
572        return False
573
574    # Commit changeset if necessary
575    if commit==1:
576        session.commit()
577       
578    return True
579
580@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
581def createCfengineHostKeys(session_id, host_id):
582    """Creates a set of Cfengine keys for the specified host"""
583    from crcnetd.modules.ccs_host import ccs_host
584    session = getSessionE(session_id)
585   
586    host = ccs_host(session_id, host_id)
587   
588    # Check for existing changeset
589    commit = 0
590    if session.changeset == 0:
591        session.begin("Created CFengine keys for %s" % host["host_name"])
592        commit = 1
593   
594    try:
595        # Determine directories to store things in
596        workdir = session.revision.getWorkingDir()
597        cfkeydir = "%s/ppkeys" % (workdir)
598        ensureDirExists(cfkeydir)
599       
600        # Generate the keys
601        genCfKey(cfkeydir, host["ip_address"])
602    except:
603        (type, value, tb) = sys.exc_info()
604        log_error("Failed to generate CFengine keys for %s! - %s" % \
605                (host["host_name"], value))
606        # Rollback any changes
607        if commit == 1:
608            session.rollback()
609        return False
610
611    # Commit changeset if necessary
612    if commit==1:
613        session.commit()
614       
615    return True
616
617def genSSHKey(dir, type):
618    """Generates a SSH host key of the specified type
619
620    dir specifies the directory to place the key in
621    """
622    filename = "ssh_host_%s_key" % type
623    key = "%s/%s" % (dir, filename)
624    pubkey = "%s.pub" % key
625   
626    # Try and avoid needless recreating keys
627    if os.path.exists(key) and os.path.exists(pubkey):
628        log_debug("Skipping SSH key creation in %s. " \
629                "Keys already exist!" % dir)
630        return
631    try:
632        os.remove(key)
633        os.remove(pubkey)
634    except:
635        pass
636       
637    fh = os.popen("cd %s &>/dev/null && /usr/bin/ssh-keygen -q -f %s " \
638            "-N '' -t %s || echo \"unable to access %s\"" % \
639            (dir, filename, type, dir))
640    output = fh.readlines()
641    rv = fh.close()
642    if rv != None or (len(output)>0 and output[0].startswith("unable")):
643        raise ccs_cfengine_error("Could not create SSH %s host key: %s" % \
644                (type, "".join(output)))
645   
646def genCfKey(dir, host_ip):
647    """Generates a Cfengine host key pair
648
649    dir specifies the directory to place the key in
650    """
651    filename = "root-%s" % host_ip
652    key = "%s/%s.priv" % (dir, filename)
653    pubkey = "%s/%s.pub" % (dir, filename)
654
655    # Try and avoid needless recreating keys
656    if os.path.exists(key) and os.path.exists(pubkey):
657        log_debug("Skipping Cfengine key creation in %s. " \
658                "Keys already exist!" % dir)
659        return
660    try:
661        os.remove(key)
662        os.remove(pubkey)
663    except:
664        pass
665
666    fh = os.popen("cd %s &>/dev/null && /usr/sbin/cfkey -f %s || echo " \
667            "\"unable to access %s\"" % (dir, filename, dir))
668    output = fh.readlines()
669    rv = fh.close()
670    if rv != None or (len(output)>0 and output[0].startswith("unable")):
671        raise ccs_cfengine_error("Could not create CFengine host key: %s" % \
672                "".join(output))
673   
674#####################################################################
675# Functions to maintain / deal with checked out Cfengine configs
676#####################################################################
677@catchEvent("revisionCreated")
678def updateCfInputDirCB(eventName, host_id, session_id, **kwargs):
679    """Callback function to trigger updates when revisions are created"""
680   
681    autoupdate = int(config_get("cfengine", "autoupdate", "0"))
682    if autoupdate:
683        updateCfInputDir(session_id)
684   
685@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
686def updateCfInputDir(session_id, revNum=-1):
687    """Updates the cfengine input directory.
688
689    Triggered explicitly by user or on the revisionCreated hook.
690    """
691   
692    svn = pysvn.Client()       
693
694    # See if there is a checked out revision
695    gotrev = 0
696    try:
697        list = svn.ls(_cfInputDir)
698        gotrev = 1
699    except:
700        # No checked out revision, remove bogus directory
701        removeDir(_cfInputDir)
702        ensureDirExists(_cfInputDir)
703        pass
704
705    if revNum == -1:
706        # Use head revision
707        revH = pysvn.Revision(pysvn.opt_revision_kind.head)
708    else:
709        # Use specified revision
710        revH = pysvn.Revision(pysvn.opt_revision_kind.number, revNum)
711   
712    if gotrev:
713        # Update the directory
714        rev = svn.update(_cfInputDir, True, revH)
715    else:
716        # Checkout the directory
717        rev = svn.checkout("%s/inputs" % _svnroot, _cfInputDir, True, revH)
718   
719    log_info("Updated cfengine configuration to revision %s." % rev.number)
720
721#####################################################################
722# Cfengine Resource Server
723#####################################################################
724class cfserver(resource.Resource):
725    """Implements a simple webserver to retrieve resources related to cfengine
726
727    Called when the main server receives a query for /cfengine/*
728
729    Currently supports the following resources
730    /cfengine/update.conf/<host_name>
731        Retrieves an update.conf file for the specified host
732    /cfkey/keyname
733        Retrieves the specified cfengine key
734       
735    Requests to this resource are restricted to the IP address ranges
736    specified in the allow_requests_from configuration directive.
737    """
738   
739    resourceName = "CFengine Resource Server"
740
741    # Mark as leaf so that render gets called
742    isLeaf = 1
743   
744    def render(self, request):
745
746        # XXX: Verify request IP
747       
748        log_debug("Received cfengine request for %s" % request.path)
749       
750        parts = request.path.split("/")
751        path = "/".join(parts[2:])
752       
753        # Process request
754        if path.startswith("update.conf"):
755            return self.getUpdateConf(path, request)
756        elif path.startswith("cfkey"):
757            return self.getCfkey(path, request)
758        elif path.startswith("hostvars"):
759            return self.getHostvars(path, request)
760        else:
761            request.setResponseCode(404, "Unknown cfengine resource requested!")
762            request.finish()
763            return server.NOT_DONE_YET
764
765    def getCfkey(self, path, request):
766        """Returns the contents of a cfengine key for the specified host."""
767       
768        parts = path.split("/")
769       
770        # Get a revision to grab the key from
771        revision = ccs_revision()
772        keydir = "%s/ppkeys" % revision.getWorkingDir()
773       
774        # Retrieve the requested key
775        keyfile = "%s/%s" % (keydir, parts[1])
776
777        try:
778            fd = open(keyfile, "r")
779            content = fd.readlines()
780            fd.close()
781        except:
782            log_warn("Could not open Cfengine key file!", sys.exc_info())
783            request.setResponseCode(404, "Unknown cfengine key requested!")
784            request.finish()
785            return server.NOT_DONE_YET
786
787        return "".join(content)
788
789    def getUpdateConf(self, path, request):
790        """Returns the contents of an update.conf file for the specified host.
791
792        Currently the host parameter is ignored.
793        """
794       
795        # Get a revision to grab the update.conf from
796        revision = ccs_revision()
797        revdir = revision.getConfigBase()
798       
799        try:
800            fd = open("%s/cfconf/update.conf" % revdir, "r")
801        except:
802            request.setResponseCode(503, "update.conf unavailable. Please " \
803                    "try again later")
804            request.finish()
805            return server.NOT_DONE_YET
806       
807        contents = fd.readlines()
808        fd.close()
809       
810        return "".join(contents)
811   
812    def getHostvars(self, path, request):
813        """Returns the contents of an update.conf file for the specified host.
814
815        Currently the host parameter is ignored.
816        """
817       
818        parts = path.split("/")
819        if len(parts)<2:
820            request.setResponseCode(404, "Hostname not specified" % host)
821            request.finish()
822            return server.NOT_DONE_YET
823        host = parts[1]
824       
825        # Get a revision to grab the hostvars file from
826        revision = ccs_revision()
827        revdir = revision.getConfigBase()
828       
829        try:
830            fd = open("%s/hosts/%s/cf.hostvars" % (revdir, host), "r")
831        except:
832            request.setResponseCode(404, "%s hostvars not found!" % host)
833            request.finish()
834            return server.NOT_DONE_YET
835       
836        contents = fd.readlines()
837        fd.close()
838       
839        return "".join(contents)
840   
841#####################################################################
842# Revision Class
843#####################################################################
844class ccs_revision:
845    """Wrapper for a revision of files.
846   
847    This class wraps the generation of an entire revision of the configuration
848    files managed by this system.
849   
850    It contains the methods needed to generate the files, insert them into
851    version control (svn) and then update then pass the resulting revision
852    identifier back to it's caller.
853    """
854
855    # Location of the svnroot
856    svnroot = None
857   
858    def __init__(self, parentSession=None, changeset=None, checkout=True):
859        """Creates a ccs_revision class.
860
861        If parentSession or changeset are not specified or None, a read-only
862        revision is created. This is only useful if you want to inspect the
863        repository without making any changes.
864        """
865       
866        self.mParentSession = parentSession
867        self.mChangeset = changeset
868        self.checkout = checkout
869       
870        # Get a SVN client to use for this revision
871        self.mSvn = pysvn.Client()
872       
873        if self.checkout:
874            # Setup a working directory for this revision
875            self.rDir = mkdtemp("", "ccsd")
876       
877            # Checkout the current configuration HEAD to this directory
878            rev = self.mSvn.checkout(self.svnroot, self.rDir)
879            self.mCurRev = rev
880       
881            # Check basic repository structure
882            if self.mParentSession is not None and self.mChangeset is not None:
883                self.checkRepoStructure()
884        else:
885            self.rDir = None
886            self.mCurRev = None
887           
888        # Start with no errors
889        self.mErrors = {}
890       
891    def __del__(self):
892       
893        # Nothing to check if nothing was checked out
894        if self.rDir is None:
895            return
896
897        # Don't check the status of a read-only revision
898        if self.mParentSession is None or self.mChangeset is None:
899            removeDir(self.rDir)
900            return
901       
902        # Check status of working directory
903        try:
904            status = self.mSvn.status(self.rDir, True, True)
905        except pysvn.ClientError:
906            # Probably never got checked out
907            status = []           
908        flag=0
909        for entry in status:
910            if entry.text_status == pysvn.wc_status_kind.ignored:
911                continue
912            if entry.text_status != pysvn.wc_status_kind.normal:
913                log_debug("File %s modified (%s) in changeset %s" % \
914                        (entry.path, entry.text_status, self.mChangeset))
915                flag=1
916       
917        if flag==1:
918            log_warn("Revision object for changeset %s in " \
919                    "session %s deleted before checkin. Changes lost!" % \
920                    (self.mChangeset, self.mParentSession.session_id))
921               
922        # Clean up the working directory
923        removeDir(self.rDir)
924       
925    def getWorkingDir(self):
926        """Returns the path that the repository is checked out into"""
927
928        return self.rDir
929   
930    def getConfigBase(self):
931        """Returns the path that cfengine config files should live in"""
932        if self.rDir is None:
933            return None
934        return "%s/inputs" % self.rDir
935   
936    def _checkForModified(self, cDir):
937        """Performs svn actions on changed files in the specified directory"""
938       
939        # Nothing to check if no repository is checked out
940        if self.rDir is None:
941            return None
942       
943        # Don't check the status of a read-only revision
944        if self.mParentSession is None or self.mChangeset is None:
945            log_warn("Cannot check status on a read-only revision!")
946            return
947       
948        # Recurse through if we were passed a list
949        if type(cDir) == type([]):
950            for d in cDir:
951                self._checkForModified(d)
952            return
953
954        # Check state
955        status = self.mSvn.status(cDir, True, True)
956        for entry in status:
957            if entry.text_status == pysvn.wc_status_kind.unversioned:
958                log_debug("Added %s in changeset %s" % \
959                        (entry.path, self.mChangeset))
960                self.mSvn.add(entry.path, False)
961                # Recurse if we added a directory
962                if os.path.isdir(entry.path):
963                    self._checkForModified(entry.path)
964            elif entry.text_status == pysvn.wc_status_kind.modified or \
965                    entry.text_status == pysvn.wc_status_kind.added or \
966                    entry.text_status == pysvn.wc_status_kind.ignored or \
967                    entry.text_status == pysvn.wc_status_kind.deleted:
968                # Don't need to mention modified files that are already
969                # processed
970                pass
971            elif entry.text_status != pysvn.wc_status_kind.normal:
972                log_debug("%s (%s) has bad state in changeset %s!" % \
973                        (entry.path, entry.text_status, self.mChangeset))
974       
975    def propset(self, path, prop, value):
976        """Sets the specified property on the specified path"""
977       
978        if self.mParentSession is None or self.mChangeset is None:
979            raise ccs_revision_error("Cannot set property on read-only " \
980                    "revision")
981       
982        # Use the svn repo directly if nothing is checked out
983        if self.rDir is None:
984            base = self.svnroot
985        else:
986            base = self.rDir
987               
988        try:
989            if not path.startswith(base):
990                filename = "%s/%s" % (base, path)
991            else:
992                filename = path
993            rev = self.mSvn.propset(prop, value, filename)
994            if self.rDir is None:
995                self.saveRevProps(rev)
996        except:
997            log_error("Failed to set property %s to %s on %s" % \
998                    (prop, value, filename))
999
1000    @registerEvent("revisionCreated")
1001    def checkin(self, message, paths=None):
1002        """Checks in the changes to the repository with the specified message"""
1003       
1004        if self.rDir is None:
1005            raise ccs_revision_error("Cannot checkin. Not checked out!")
1006       
1007        if self.mParentSession is None or self.mChangeset is None:
1008            raise ccs_revision_error("Cannot checkin a read-only revision")
1009       
1010        # Default to the whole repository
1011        if paths is None:
1012            paths = self.rDir
1013           
1014        # Check status of working directory and add / del files etc
1015        self._checkForModified(paths)
1016
1017        r = self.mSvn.checkin(paths, message, True)
1018        n = self.saveRevProps(r)
1019        if n<0:
1020            # Nothing changed
1021            return -1
1022       
1023        triggerEvent(self.mParentSession.session_id, "revisionCreated", \
1024                revision_no=n)
1025
1026        return n
1027
1028    def saveRevProps(self, r):
1029        """Saves customised properties against each revision
1030
1031        These properties are used to help keep track of how the database/svn
1032        repository changes match up.
1033        """
1034       
1035        if r is None:
1036            # Nothing changed
1037            return -1
1038
1039        # Set the author property on the checkin
1040        try:
1041            r2 = self.mSvn.revpropset("svn:author", \
1042                    self.mParentSession.username, self.svnroot, r)
1043        except:
1044            (type, value, tb) = sys.exc_info()
1045            log_warn("Could not set author property on revision %s - %s" % \
1046                    (r.number, value))
1047            r2 = r
1048       
1049        # Record the changeset that triggered this revision
1050        try:
1051            r3 = self.mSvn.revpropset("ccs:changeset", "%s" % \
1052                    self.mChangeset, self.svnroot, r2)
1053        except:
1054            (type, value, tb) = sys.exc_info()
1055            log_warn("Could not set changeset property on " \
1056                    "revision %s - %s" % (r2.number, value))
1057
1058        log_info("Committed revision %s to version control" % r2.number)
1059        return r2.number
1060   
1061    def fileExists(self, path):
1062        """Checks if the specified file exists in the repository"""
1063       
1064        # Use the svn repo directly if nothing is checked out
1065        if self.rDir is None:
1066            base = self.svnroot
1067        else:
1068            base = self.rDir
1069               
1070        try:
1071            if not path.startswith(base):
1072                filename = "%s/%s" % (base, path)
1073            else:
1074                filename = path
1075            e = self.mSvn.ls(filename)
1076            if len(e) > 0:
1077                return True
1078        except:
1079            log_warn("Failed to check existance of file: %s" % filename)
1080            return False
1081       
1082        return False
1083       
1084    def checkRepoStructure(self):
1085        """Checks the repository has all the required base directories.
1086
1087        If a base directory is not present it is created. The base directories
1088        are:
1089        inputs/        Host/Service configuration files
1090
1091        Further hierarchy within each base directory is the responsibility of
1092        other modules.
1093        """
1094       
1095        if self.rDir is None:
1096            raise ccs_revision_error("Cannot check repository structure. " \
1097                    "Not checked out!")
1098       
1099        if self.mParentSession is None or self.mChangeset is None:
1100            log_warn("Cannot check repository structure on a read-only " \
1101                    "revision")
1102            return
1103       
1104        flag = 0
1105       
1106        configsDir = self.getConfigBase()
1107        n = ensureDirExists(configsDir)
1108        if n > 0:
1109            # Schedule Additions
1110            bDir = configsDir
1111            while n>1:
1112                bDir = os.path.dirname(configsDir)
1113                n-=1
1114            log_info("Created configuration directory (%s) within " \
1115                    "repository" % configsDir)
1116            self.mSvn.add(bDir, True)
1117            flag = 1
1118       
1119        # Commit directory changes immediately
1120        if flag==1:
1121            r = self.mSvn.checkin(self.rDir, \
1122                    "checkRepoStructure created missing directories", True)
1123
1124def validateRepository(svnroot):
1125    """Checks for the existance of a valid svn repository at svnroot"""
1126
1127    # Extract the path from the URL
1128    svnpath = svnroot[svnroot.find("://")+3:]
1129
1130    # Check directory exists
1131    if not os.path.exists(svnpath):
1132        log_debug("Specified repository path does not exist: %s" % svnpath)
1133        return False
1134   
1135    # Check it looks a bit like a subversion repository
1136    fp = open("%s/README.txt" % svnpath, "r")
1137    if not fp:
1138        log_debug("No README.txt inside repository: %s" % svnpath)
1139        return False
1140    tmp = fp.read().strip()
1141    fp.close()
1142    if not tmp.startswith("This is a Subversion repository"):
1143        log_debug("invalid README.txt in repository: %s" % svnpath)       
1144        return False
1145
1146    # Final, master check, can we check it out
1147    tmpdir = mkdtemp("", "ccsd")
1148    try:
1149        client = pysvn.Client()
1150        rev = client.checkout(svnroot, tmpdir)
1151    except:
1152        log_error("Unable to checkout from svn repository at %s" % svnroot)
1153        removeDir(tmpdir)
1154        return False
1155    removeDir(tmpdir)
1156
1157    # All OK
1158    return True
1159
1160def createRepository(svnroot):
1161    """Initialises a blank repository and configures it for use by ccsd"""
1162
1163    # Look for the svnadmin utility
1164    fd = os.popen("/usr/bin/which svnadmin 2>/dev/null")
1165    path = fd.read().strip()
1166    if fd.close() != None:
1167        log_error("Cannot find svnadmin utility.")
1168        return False
1169   
1170    # Check we can execute the svnadmin utilty
1171    if not os.access(path, os.X_OK):
1172        log_error("Cannot execute svnadmin utility.")
1173        return False
1174
1175    # Try and create the repository
1176    svnpath = svnroot[svnroot.find("://")+3:]
1177    fd = os.popen("%s create --fs-type fsfs %s 2>&1" % (path, svnpath))
1178    result = fd.read().strip()
1179    if fd.close() != None:
1180        log_error("svnadmin create failed. See tb log for more details.")
1181        log_tb(None, result)
1182        return False
1183   
1184    # Add a pre-revprop-change hook to allow the daemon to set author
1185    # revision properties
1186    hook = "%s/hooks/pre-revprop-change" % svnpath
1187    fd = open(hook, "w")
1188    if not fd:
1189        log_warning("Unable to setup pre-revprop-change hook in new " \
1190                "repository!")
1191    else:
1192        fd.write("""#!/bin/bash
1193
1194# PRE-REVPROP-CHANGE HOOK
1195
1196# Created by the CRCnet Configuration System Daemon on
1197# %s
1198
1199# Allow all changes at this time
1200exit 0
1201""" % time.ctime())
1202        fd.close()
1203        os.chmod(hook, 0755)
1204
1205    # Success
1206    log_info("Subversion repository created at %s" % svnpath)
1207    return True
1208
1209#####################################################################
1210# Cfengine Integration Initialisation
1211#####################################################################
1212_hostTemplates = {}
1213_networkTemplates = {}
1214_templateDir = ""
1215_templateModDir = ""
1216_events = {}
1217_cfBaseDir = ""
1218_cfInputDir = ""
1219_svnroot = ""
1220 
1221def initCFengine():
1222    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
1223    global _events, _cfBaseDir, _cfInputDir, _svnroot
1224   
1225    # Retrieve template directory configuration
1226    _templateDir = config_get("cfengine", "template_dir")
1227    if _templateDir == None or _templateDir == "":
1228        log_fatal("Invalid template directory (%s)!" % _templateDir)
1229    _templateModDir = config_get("cfengine", "template_module_dir")
1230    if _templateModDir == None or _templateModDir == "":
1231        log_fatal("Invalid template module directory (%s)!" % _templateModDir)
1232
1233    # Retrieve cfengine configuration file directory configuration
1234    _cfBaseDir = config_get("cfengine", "cfbase_dir")
1235    if _cfBaseDir == None or _cfBaseDir == "":
1236        log_fatal("Invalid cfengine base directory (%s)!" % _cfBaseDir)
1237    _cfInputDir = "%s/inputs" % _cfBaseDir
1238
1239    # Decide where the svn root is
1240    _svnroot = config_get("cfengine", "config_svnroot")     
1241    if _svnroot == None or _svnroot == "":
1242        log_fatal("Configuration Data subversion repository not specified!")
1243    # Check that the svnroot is a valid svn URL
1244    if _svnroot.find("://")==-1:
1245        log_fatal("Invalid svnroot URL: %s" % _svnroot)   
1246    # Validate the repository and create it if it doesn't exist
1247    if not validateRepository(_svnroot):
1248        log_warn("No svn repository at specified path. Attempting " \
1249                "to create one.")
1250        if not createRepository(_svnroot):
1251            log_fatal("Could not create repository at %s" % _svnroot)
1252    # Let the revision class know where to get its data from
1253    ccs_revision.svnroot = _svnroot
1254   
1255    # Setup template variables from configuration file
1256    ccs_template.domain = config_get_required("network", "domain")
1257    ccs_template.site_name = config_get_required("network", "site_name")
1258    ccs_template.server_name = config_get_required("network", "server_name")
1259    ccs_template.smtp_server = config_get_required("network", "smtp_server")
1260    ccs_template.admin_email = config_get_required("network", "admin_email")
1261   
1262    # Initialise the templates
1263    initTemplates()
1264   
1265    # Register Cfengine Resource Server
1266    registerResource("cfengine", cfserver)
Note: See TracBrowser for help on using the repository browser.