source: ccsd/trunk/crcnetd/_utils/ccsd_cfengine.py @ 1024

Last change on this file since 1024 was 1024, checked in by mglb1, 7 years ago

Gracefully handle the lack of status information for a host by creating
a status object that isn't going to be updated
  • Property svn:keywords set to Id
File size: 81.3 KB
Line 
1# Copyright (C) 2006  The University of Waikato
2#
3# This file is part of crcnetd - CRCnet Configuration System Daemon
4#
5# CFengine Configuration Setup
6#
7# Manages the cfengine configuration for the configuration system. The two
8# primary tasks involved in this are:
9# - Configuration file generation from templates
10# - Controling cfengine runs and collecting output on demand
11#
12# Author:       Matt Brown <matt@crc.net.nz>
13# Version:      $Id$
14#
15# crcnetd is free software; you can redistribute it and/or modify it under the
16# terms of the GNU General Public License version 2 as published by the Free
17# Software Foundation.
18#
19# crcnetd is distributed in the hope that it will be useful, but WITHOUT ANY
20# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
22# details.
23#
24# You should have received a copy of the GNU General Public License along with
25# crcnetd; if not, write to the Free Software Foundation, Inc.,
26# 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
27import sys
28import os
29import os.path
30from stat import *
31import pysvn
32import imp
33import time
34import threading
35from twisted.web import resource, server
36from tempfile import mkdtemp
37from Cheetah.Template import Template
38
39from crcnetd._utils.ccsd_common import *
40from crcnetd._utils.ccsd_log import *
41from crcnetd._utils.ccsd_events import *
42from crcnetd._utils.ccsd_config import config_get, config_get_required
43from crcnetd._utils.ccsd_session import getSession, getSessionE
44from crcnetd._utils.ccsd_server import registerResource, exportViaXMLRPC, \
45        initThread, registerRecurring, suggestThreadpoolSize
46
47DEFAULT_MAX_THREADS = 10
48
49class ccs_template_error(ccsd_error):
50    pass
51class ccs_cfengine_error(ccsd_error):
52    pass
53
54#####################################################################
55# General Cfengine Integration Helper Functions
56#####################################################################
57def initTemplates():
58    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
59   
60    # Ensure output template module directories exist
61    ensureDirExists("%s/host" % _templateModDir)
62    ensureDirExists("%s/network" % _templateModDir)
63
64    # Ensure templates are compiled
65    rv = os.system("/usr/bin/cheetah compile -R --idir %s/host --odir " \
66            "%s/host --nobackup &>/dev/null" % (_templateDir, _templateModDir))
67    if rv != 0:
68        log_error("Unable to compile all host templates!")
69    rv = os.system("/usr/bin/cheetah compile -R --idir %s/network --odir " \
70            "%s/network --nobackup &>/dev/null" % \
71            (_templateDir, _templateModDir))
72    if rv != 0:
73        log_error("Unable to compile all network templates!")
74   
75    # Load template files
76    _hostTemplates = loadTemplates("%s/host" % _templateModDir)
77    log_info("Loaded %s host templates for cfengine." % len(_hostTemplates))
78    _networkTemplates = loadTemplates("%s/network" % _templateModDir)
79    log_info("Loaded %s network templates for cfengine." % \
80            len(_networkTemplates))
81
82    # Register for events
83    registerEvents(_hostTemplates)
84    registerEvents(_networkTemplates)
85
86def loadTemplates(moduleDir, baseStrip=""):
87    """Creates a dictionary of template modules from the specified dir"""
88   
89    # Dictionary of template objects
90    templates = {}
91   
92    if baseStrip == "":
93        baseStrip = moduleDir
94       
95    # Get a list of possible templates
96    if os.access(moduleDir, os.R_OK) != 1:
97        raise ccs_template_exception("Unable to access directory! - %s" % \
98                moduleDir)
99    ptempls = os.listdir(moduleDir)
100   
101    # Scan through the list and load valid modules
102    for tfile in ptempls:
103        # Ignore hidden files
104        if tfile.startswith("."):
105            continue
106        tfilename = "%s/%s" % (moduleDir, tfile)
107        # Recurse into directories
108        if os.path.isdir(tfilename):
109            templates.update(loadTemplates(tfilename, baseStrip))
110            continue
111        # Ignore non module files
112        if not tfile.endswith(".py"):
113            continue
114        # Load the template module
115        tname = os.path.basename(tfilename)[:-3]
116        if tname == "__init__":
117            # Ignore python framework stuff
118            continue
119        m = None
120        try:
121            m = imp.load_source(tname, tfilename)
122            # Don't want the module in the system module list
123            if tname in sys.modules.keys():
124                del sys.modules[tname]
125        except:
126            log_debug("Module import failed for %s template" % tname, \
127                    sys.exc_info())
128            pass
129        if not m:
130            log_error("Failed to import template: %s" % tname)
131            continue
132        # If an explicit output filename was specified use that, otherwise
133        # prepend path information and store the template filename.
134        path = "%s/" % os.path.dirname(tfilename[len(baseStrip)+1:])
135        if len(path) == 1:
136            path = ""
137        tclass = eval("m.%s" % tname)
138        m.fileName = "%s%s" % (path, getattr(tclass, "fileName", tname))
139        m.multiFile = getattr(tclass, "multiFile", False)
140        m.templateName = tname
141        # Store the template for future use, prepend path for uniqueness
142        templateID = "%s%s" % (path, tname)
143        if templateID in templates.keys():
144            log_error("Could not import duplicate template: %s" % tname)
145            continue
146        templates[templateID] = m
147   
148    # Return the templates
149    return templates
150
151def registerEvents(templates):
152    """Runs through the list of templates and registers callbacks"""
153   
154    for templateID, template in templates.items():
155        events = eval("template.%s.eventList" % template.templateName)
156        if events == [] or not events:
157            continue
158        for event in events:
159            registerCallback(templateID, event)
160   
161def registerCallback(templateID, eventName):
162    """Keeps our internal list of what templates want to be called"""
163
164    if eventName in _events.keys():
165        _events[eventName].append(templateID)
166        return
167
168    try:
169        catchEvent(eventName, processEvent)
170        _events[eventName] = [templateID]
171    except ccs_event_error:
172        log_warn("Template %s not bound to event %s!" % \
173                (templateID, eventName), sys.exc_info())
174
175def processEvent(eventName, host_id, session_id, **params):
176    """Callback process from the event module.
177   
178    This function is called when an event that a template has indicated
179    an interest in is fired. Look through our templates to find which
180    one is interested and call it.
181    """
182    global _hostTemplates, _networkTemplates
183    from crcnetd.modules.ccs_host import getHostName
184   
185    # Check session to see if a revision is active
186    session = getSessionE(session_id)
187    if session.revision == None:
188        return
189
190    host_name = getHostName(session_id, host_id)
191    hosts = {host_name:[],"network":[]}
192    # Fire all the templates attached to this event
193    for templateID in _events[eventName]:
194        # Process host templates if applicable
195        if host_id != -1 and templateID in _hostTemplates.keys():
196            hosts[host_name].append(templateID)
197        # Process any network templates
198        if templateID in _networkTemplates.keys():
199            hosts["network"].append(templateID)
200
201    # Fire off the generation job
202    generateTemplate(session_id, hosts)
203   
204@registerRecurring(60*10)
205def cleanTemplateStatus():
206    """Runs every ten minutes to clean up expired template status info"""
207    global _templateStats, _statsLock
208   
209    # If generation is not finished by this many minutes after initiation
210    # there is an error, stats are removed in hope of causing a Key error
211    # that will kill the errant threads
212    err_time = 60*60
213    # Statistics for finished generation runs are removed this many minutes
214    # after the run completes
215    expire_time = 60*30
216   
217    _statsLock.acquire()
218    try:
219        for key,stats in _templateStats.items():
220            if (time.time()-stats["initiated"]) > err_time:
221                # Still processing after one hour!?
222                log_error("Template generation (%s) was still active after " \
223                        "%d seconds!" % (key, err_time))
224                del _templateStats[key]
225                continue
226            if stats["finished"] == 0:
227                # Still in progress
228                continue
229            if (time.time()-stats["finished"]) > expire_time:
230                log_info("Removing template generation status (%s)" % key)
231                del _templateStats[key]
232    finally:
233        _statsLock.release()
234
235    return True
236   
237@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
238def getTemplateStatus(session_id, statsKey):
239    """Returns the status of a specified template generation run"""
240    global _templateStats, _statsLock
241
242    stats = None
243   
244    _statsLock.acquire()
245    try:
246        if statsKey not in _templateStats.keys():
247            raise ccs_cfengine_error("Invalid key: %s" % statsKey)
248        stats = dict(_templateStats[statsKey])
249    finally:
250        _statsLock.release()
251
252    return stats
253
254@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
255def generateTemplate(session_id, requested_templates={}):
256    """Generates the specified configuration templates
257
258    The requested_templates parameter should be a dictionary indexed by
259    host_name each entry should contain a list of the template ids that
260    should be generated for that host.
261   
262    If the template list for a host is empty all available templates will be
263    regenerated.
264    If the dictionary is completely empty then all hosts will have their
265    templates regenerated.
266
267    If an entry in the dictionary contains the key "network" it will be
268    interpreted as a list of network wide templates to regenerate. If this key
269    is absent then the network wide templates will be regenerated if the number
270    of host templates being regenerated is greater than one.
271   
272    All template processing is shunted off to another thread and the user is
273    returned a token that can be passed to future calls to getTemplateStatus
274    to retrieve the current status of template generation. Generation results
275    are stored for 30 minutes after the end of template generation.
276    """
277    global _hostTemplates, _networkTemplates, _templateStats, _statsLock
278    from crcnetd.modules.ccs_host import getHostList
279
280    # Setup the basic set of statistics about template generation
281    stats = {}
282    stats["planned"] = {}
283    stats["generated"] = {}
284    stats["initiated"] = time.time()
285    stats["setupprogress"] = 10
286    stats["generating"] = 0
287    stats["finished"] = 0
288    stats["error"] = None
289
290    # Work out what the user has requested us to generate
291    hosts = {}
292    host_names = requested_templates.keys()
293    if len(host_names) == 0:
294        host_names = [host["host_name"] for host in getHostList(session_id)]
295    for host in host_names:
296        if host == "network": continue
297        templates = []
298        if host in requested_templates.keys():
299            templates = requested_templates[host]
300        if len(templates) == 0:
301            templates = _hostTemplates.keys()
302        hosts[host] = templates
303    stats["planned"]["hosts"] = hosts
304    stats["generated"]["hosts"] = {}
305   
306    # Work out how many host templates we're going to be generating
307    total = 0
308    for tlist in hosts.values():
309        total += len(tlist)
310    stats["planned"]["total"] = total
311   
312    # Add network level templates if more than one host template is requested
313    # or they have been explicitly specified
314    if "network" in host_names:
315        stats["planned"]["network"] = requested_templates["network"]
316    elif total > 1:
317        stats["planned"]["network"] = _networkTemplates.keys()
318    else:
319        stats["planned"]["network"] = []
320    stats["planned"]["total"] += len(stats["planned"]["network"])
321    stats["generated"]["network"] = {}
322       
323    # Aquire the lock to deal with statistics
324    statsKey = token = createPassword(8)
325    _statsLock.acquire()
326    try:
327        _templateStats[statsKey] = stats
328    finally:
329        _statsLock.release()
330
331    # Fire off the new thread
332    initThread(processTemplates, session_id, statsKey)
333
334    # Return back to the user
335    return statsKey
336   
337def processTemplates(session_id, statsKey):
338    """Does the hardwork of generating templates in a thread"""
339    global _templateStats, _statsLock, _threadLimit
340    from crcnetd.modules.ccs_host import getHostList, ccs_host, \
341            getDistributions
342    from crcnetd._utils.ccsd_service import getServiceTemplateVars
343    from crcnetd.modules.ccs_asset import getAssetTypeTemplateVariables
344
345    log_info("Entered template processing thread (%s)" % statsKey)
346   
347    # Make sure we have a revision to insert files into
348    # Check session to see if a revision is active
349    session = getSessionE(session_id)
350    commit=0
351    if session.revision == None:
352        session.begin("Updated host configuration files", initiator="cfengine")
353        commit=1
354    outputDir = session.revision.getConfigBase()
355   
356    # More progress
357    _statsLock.acquire()
358    try:
359        _templateStats[statsKey]["setupProgress"] = 15
360    finally:
361        _statsLock.release()
362       
363    # Load all the data that we need to pass to the templates
364    variables = {}
365    hlist = {}
366    id2name = {}
367    for host in getHostList(session_id):
368        host = ccs_host(session_id, host["host_id"])
369        hostDetails = host.getTemplateVariables()
370        hlist[host["host_name"]] =  hostDetails
371        _statsLock.acquire()
372        try:
373            if _templateStats[statsKey]["setupProgress"] < 70:
374                _templateStats[statsKey]["setupProgress"] += 2
375        finally:
376            _statsLock.release()
377    variables["hosts"] = hlist
378    dlist = {}
379    for distrib in getDistributions(session_id):
380        dlist[distrib["distribution_id"]] = distrib
381    variables["distributions"] = dlist
382    variables["services"] = getServiceTemplateVars(session_id)
383    _statsLock.acquire()
384    try:
385        _templateStats[statsKey]["setupProgress"] = 80
386    finally:
387        _statsLock.release()
388    variables["date"] = time.ctime()
389    variables["domain"] = config_get_required("network", "domain")
390    variables["site_name"] = config_get_required("network", "site_name")
391    variables["server_name"] = config_get_required("network", "server_name")
392    variables["policy_ip"] = getIP(variables["server_name"])
393    variables["smtp_server"] = config_get_required("network", "smtp_server")
394    variables["admin_email"] = config_get_required("network", "admin_email")
395    _statsLock.acquire()
396    try:
397        _templateStats[statsKey]["setupProgress"] = 90
398    finally:
399        _statsLock.release()
400
401    variables["asset_types"] = getAssetTypeTemplateVariables(session_id)
402    variables["session_id"] = session_id
403
404    # Update the stats and get the list of hosts
405    _statsLock.acquire()
406    try:
407        planned = _templateStats[statsKey]["planned"]
408        _templateStats[statsKey]["generating"] = time.time()
409        _templateStats[statsKey]["setupProgress"] = 100
410    finally:
411        # Release the lock before proceeding
412        _statsLock.release()
413   
414    # Now generate each hosts template in a separate thread
415    for host,template_ids in planned["hosts"].items():
416        # See if we can start a new thread
417        _threadLimit.acquire()
418        try:
419            # Extract the host specific data for this template
420            hostData = variables["hosts"][host]
421            # Fire off the thread to process this host, the thread will release
422            # the semaphore as it is about to exit
423            # XXX: This relies on the thread behaving nicely, need to find a
424            # way to find out if it's been bad and exited without releasing
425            # the semaphore
426            initThread(processHostTemplates, statsKey, outputDir, hostData, \
427                    variables)
428        except:
429            log_error("Failed to start thread to process templates for " \
430                    "host: %s!" % host, sys.exc_info())
431            _threadLimit.release()
432           
433    # Wait until all the configuration files have been generated
434    hosts = list(planned["hosts"].keys())
435    hasError = False
436    hasTemplateError = False
437    while len(hosts) > 0:
438        # Wait a bit before trying again
439        time.sleep(1)
440        _statsLock.acquire()
441        try:
442            generated = _templateStats[statsKey]["generated"]["hosts"]
443            for host in hosts:
444                if not host in generated.keys():
445                    continue
446                # Check if it finished successfully
447                if generated[host]["finished"] > 0:
448                    # Remove from list
449                    hosts.remove(host)
450                    # Check if any of the templates failed to be output
451                    if generated[host]["error"] == "template":
452                        # Set the flag so we don't commit the revision
453                        hasTemplateError = True
454                else:
455                    # Check for fatal errors
456                    if generated[host]["error"] != "":
457                        # Set the flag and remove the host from processing
458                        hosts.remove(host)
459                        hasError = True
460                        continue
461        finally:
462            _statsLock.release()
463   
464    # Hosts are done, network templates now
465    for template_id in planned["network"]:
466        # See if we can start a new thread
467        _threadLimit.acquire()
468        try:
469            # Fire off the thread to process this template, the thread will
470            # release the semaphore as it is about to exit
471            # XXX: This relies on the thread behaving nicely, need to find a
472            # way to find out if it's been bad and exited without releasing
473            # the semaphore
474            initThread(processNetworkTemplate, statsKey, outputDir, \
475                    template_id, variables)
476        except:
477            log_error("Failed to start thread to process network template: " \
478                    "%s!" % template_id, sys.exc_info())
479            _threadLimit.release()
480
481    # Wait until all network templates are done
482    network = list(planned["network"])
483    while len(network) > 0:
484        # Wait a bit before trying again
485        time.sleep(1)
486        _statsLock.acquire()
487        try:
488            generated = _templateStats[statsKey]["generated"]["network"]
489            for template in network:
490                if not template in generated.keys():
491                    continue
492                # Check for errors
493                if generated[template]["error"] != "":
494                    hasTemplateError = True
495                # Remove from the list
496                network.remove(template)
497        finally:
498            _statsLock.release()       
499   
500    # Commit if neceesary and no template/host errors occured
501    rv = {"revision":None}
502    if commit and not (hasError or hasTemplateError):
503        rv = session.commit()
504       
505    # All done
506    _statsLock.acquire()
507    try:
508        _templateStats[statsKey]["finished"] = time.time()
509        _templateStats[statsKey]["revision"] = rv["revision"]
510        d = _templateStats[statsKey]["finished"] - \
511                _templateStats[statsKey]["initiated"]
512        if hasError:
513            _templateStats[statsKey]["error"] = "host"
514        elif hasTemplateError:
515            _templateStats[statsKey]["error"] = "template"
516    finally:
517        _statsLock.release()
518
519    log_info("Template generation completed in %0.3f seconds for %s" % \
520            (d, statsKey))
521    return True
522
523def processHostTemplates(statsKey, outputDir, hostData, networkData):
524    """Runs in a thread and generates the output files for the template"""
525    global _templateStats, _statsLock, _threadLimit, _hostTemplates
526    host_id = -1
527    host = "unknown"
528   
529    # It is imperative that we release the semaphore before returning from
530    # this function or we could potentially starve the calling thread of
531    # workers and we'd end up in a deadlock situation!
532    try:
533        # Work out the host_id and host_name of this host
534        host_id = hostData["host_id"]
535        host = hostData["host_name"]
536        hostPath = "%s/hosts/%s" % (outputDir, host)
537        start = time.time()
538        log_debug("Started host generation thread for %s" % host)
539        session = getSessionE(networkData["session_id"])
540       
541        # Retrieve the list of templates to generate and flag the start
542        _statsLock.acquire()
543        try:
544            planned = _templateStats[statsKey]["planned"]
545            generated = _templateStats[statsKey]["generated"]["hosts"]
546            generated[host] = {}
547            generated[host]["initiated"] = start
548            generated[host]["finished"] = 0
549            generated[host]["error"] = ""
550        finally:
551            _statsLock.release()
552
553        # Now that we have a set of templates to use do the actual generation
554        hadError = False
555        for tname in planned["hosts"][host]:
556            try:
557                tstart = time.time()
558                # Instantiate a template
559                template = _hostTemplates[tname]
560                t = eval("template.%s()" % template.templateName)
561                # Is it enabled on this host?
562                if not t.enabledOnHost(networkData["session_id"], host_id):
563                    continue
564                # Set up the variables we want to substitute in
565                t._searchList = [hostData, networkData]
566                # Generate the file
567                filename = "%s/%s" % (hostPath, template.fileName)
568                files = t.writeTemplate(filename, template.multiFile)
569                tend = time.time()
570                # Set properties
571                format = getattr(t, "highlightFormat", "")
572                if format != "":
573                    for file in files:
574                        session.revision.propset(file, "ccs:format", format)
575                # Record how long the file took to generate
576                _statsLock.acquire()
577                try:
578                    info = {"time":tend-tstart, "error":""}
579                    generated[host][tname] = info
580                finally:
581                    _statsLock.release()
582            except:
583                (type, value, tb) = sys.exc_info()
584                # Single template errors can be skipped over, nothing will
585                # get committed unless the user explicitly approves
586                _statsLock.acquire()
587                try:
588                   
589                    info = {"time":0, "error":value}
590                    generated[host][tname] = info
591                finally:
592                    _statsLock.release()
593                hadError = True
594                log_error("Failed to process template (%s) for host: %s" % \
595                        (tname, host), (type, value, tb))
596       
597        # Clean up
598        end = time.time()
599        log_debug("Completed host generation for %s in %0.3f seconds" % \
600                (host, (end-start)))
601        _statsLock.acquire()
602        try:
603            generated[host]["finished"] = end
604            # If a template failed make a note of it here
605            if hadError:
606                generated[host]["error"] = "template"
607        finally:
608            _statsLock.release()
609    except:
610        (type, value, tb) = sys.exc_info()
611        # Record the error for display to the user
612        _statsLock.acquire()
613        try:
614            generated = _templateStats[statsKey]["generated"]["hosts"]
615            if host not in generated.keys():
616                generated[host] = {}
617                generated[host]["initiated"] = 0
618            generated[host]["finished"] = time.time()
619            generated[host]["error"] = value
620        finally:
621            _statsLock.release()
622        # Log the error for the administrators record
623        log_error("Exception while processing host templates (%s)!" % \
624                host, (type, value, tb))
625   
626    # Release the semaphore
627    _threadLimit.release()
628    return
629
630def processNetworkTemplate(statsKey, outputDir, template_id, networkData):
631    """Runs in a thread and generates the output files for the template"""
632    global _templateStats, _statsLock, _threadLimit, _networkTemplates
633   
634    # It is imperative that we release the semaphore before returning from
635    # this function or we could potentially starve the calling thread of
636    # workers and we'd end up in a deadlock situation!
637    try:
638        log_debug("Started template generation thread for %s" % template_id)
639        tstart = time.time()
640        session = getSessionE(networkData["session_id"])
641        # Instantiate a template
642        template = _networkTemplates[template_id]
643        t = eval("template.%s()" % template.templateName)
644        # Set up the variables we want to substitute in
645        t._searchList = [networkData]
646        # Generate the file
647        filename = "%s/%s" % (outputDir, template.fileName)
648        files = t.writeTemplate(filename, template.multiFile)
649        tend = time.time()
650        # Set properties
651        format = getattr(template, "highlightFormat", "")
652        if format != "":
653            for file in files:
654                session.revision.propset(file, "ccs:format", format)
655        # Record how long the file took to generate
656        _statsLock.acquire()
657        try:
658            generated = _templateStats[statsKey]["generated"]["network"]
659            info = {"time":tend-tstart, "error":""}
660            generated[template_id] = info
661        finally:
662            _statsLock.release()
663        log_debug("Completed template generation for %s in %0.3f seconds" \
664                % (template_id, (tend-tstart)))
665    except:
666        (type, value, tb) = sys.exc_info()
667        # Skip over the error, nothing will get committed unless the user
668        # explicitly approves
669        _statsLock.acquire()
670        try:
671            generated = _templateStats[statsKey]["generated"]["network"]
672            info = {"time":0, "error":value}
673            generated[template_id] = info
674        finally:
675            _statsLock.release()
676        log_error("Failed to process network template (%s)" % template_id, \
677                (type, value, tb))
678   
679    # Release the semaphore
680    _threadLimit.release()
681    return
682
683#####################################################################
684# Template Mixin
685#####################################################################
686class ccs_template(Template):
687    """Config System Template Processor
688   
689    All templates that expect to be processed by the configuration system
690    must derive from this class. It provides helper functions that are required
691    to determine where and when each template should be generated and how to
692    store it to disk.
693    """
694
695    def __init__(self):
696        """Initialise the class
697
698        You must call this method from your subclass
699        """
700        # Call Cheetah's init
701        Template.__init__(self)
702
703    def writeTemplate(self, filename, multiFile):
704        """Writes the template output to the specified file"""
705       
706        files = []
707       
708        # Ensure the output directory exists
709        ensureDirExists(os.path.dirname(filename))
710
711        # Get the template contents
712        template = self.writeBody().strip()
713        if not template.endswith("\n"): template += "\n"
714
715        # Write it out to a file
716        if not multiFile:
717            f = open(filename, "w")
718            f.write(template)
719            f.close()
720            files.append(filename)
721        else:
722            # Handle templates that generate multiple output files
723            lines = template.split("\n")
724            f = None
725            for line in lines:
726                if line.startswith(".newfile"):
727                    # New file starting, close previous file
728                    if f is not None: f.close()
729                    # Open new file
730                    parts = line.split(" ")
731                    if len(parts) != 2:
732                        raise ccs_template_error("Invalid multifile template!")
733                    fname = "%s%s" % (filename, parts[1])
734                    f = open(fname, "w")
735                    files.append(fname)
736                    continue
737                elif f is not None:
738                    # Write the line out
739                    f.write("%s\n" % line)
740            # Close the file
741            if f is not None: f.close() 
742
743        return files
744
745    def __str__(self):
746        return self.writeBody()
747   
748    def enabledOnHost(self, session_id, host_id):
749        """Returns true if the template is applicable to the specified host
750
751        By default this method looks to see if the template has defined a
752        serviceName parameter. If it has then the function checks to see if
753        that service is enabled on the specified host. If it is not the
754        function returns False.
755       
756        This function may be overriden by other classes/templates if you want
757        to implement more logic than the default implementation provides.
758        """
759        serviceName = getattr(self, "serviceName", None)
760        if serviceName is None:
761            return True
762
763        from crcnetd.modules.ccs_host import ccs_host
764        host = ccs_host(session_id, host_id)
765        return host.hasServiceEnabledByName(serviceName)
766
767    def getTemplateVariables(self):
768        """Returns a dictionary of variables that can be used by the template
769
770        The dictionary is passed to Cheetah's searchList so that it's entries
771        can be used as placeholders in the template.
772
773        This function retains an empty list. You should override this in your
774        implementing class.
775        """
776        return []
777
778#####################################################################
779# Functions to maintain / deal with hosts
780#####################################################################
781def getCfengineHostStatus(session_id, host):
782    """Returns a dictionary describing the state of the host configuration"""
783
784    status = {}
785   
786    # Check keys exist
787    revision = ccs_revision(checkout=False)
788   
789    sshkeydir = "inputs/sshkeys/%s" % host["host_name"]
790    cfkeydir = "ppkeys"
791   
792    # SSH keys
793    status["ssh_key_status"] = STATUS_OK
794    status["ssh_key_text"] = ""
795    for key in ["dsa", "rsa"]:
796        if not revision.fileExists("%s/ssh_host_%s_key" % (sshkeydir, key)):
797            status["ssh_key_status"] = STATUS_CRITICAL
798            status["ssh_key_text"] += "%s private key missing, " % \
799                    key.upper()
800        if not revision.fileExists("%s/ssh_host_%s_key.pub" % \
801                (sshkeydir, key)):
802            status["ssh_key_status"] = STATUS_CRITICAL
803            status["ssh_key_text"] += "%s private key missing, " % \
804                    key.upper()
805    if status["ssh_key_text"]!="":
806        status["ssh_key_text"]  = status["ssh_key_text"][:-2]
807   
808    # Cfengine keys
809    status["cfengine_key_status"] = STATUS_OK
810    status["cfengine_key_text"] = ""
811    if not revision.fileExists("%s/root-%s.priv" % \
812            (cfkeydir, host["ip_address"])):
813        status["cfengine_key_status"] = STATUS_CRITICAL
814        status["cfengine_key_text"] += "CFengine private key missing"
815    if not revision.fileExists("%s/root-%s.pub" % \
816            (cfkeydir, host["ip_address"])):
817        status["cfengine_key_status"] = STATUS_CRITICAL
818        if status["cfengine_key_text"]!="": status["cfengine_key_text"]+=", "
819        status["cfengine_key_text"] += "CFengine public key missing"
820
821    return status
822   
823@catchEvent("hostAdded")
824def hostAddedCB(eventName, host_id, session_id, **params):
825    """Callback function to setup host configuration upon host creation"""
826
827    session = getSession(session_id)
828    if session is None:
829        return
830
831    # Create keys for the host
832    createSSHHostKeys(session_id, host_id)
833    createCfengineHostKeys(session_id, host_id)
834
835@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
836def createSSHHostKeys(session_id, host_id):
837    """Creates a set of SSH keys for the specified host"""
838    from crcnetd.modules.ccs_host import ccs_host
839    session = getSessionE(session_id)
840   
841    host = ccs_host(session_id, host_id)
842   
843    # Check for existing changeset
844    commit = 0
845    if session.changeset == 0:
846        session.begin("Created SSH keys for %s" % host["host_name"])
847        commit = 1
848   
849    try:
850        # Determine directories to store things in
851        revdir = session.revision.getConfigBase()
852        sshkeydir = "%s/sshkeys/%s" % (revdir, host["host_name"])
853        ensureDirExists(sshkeydir)
854       
855        # Generate the keys
856        genSSHKey(sshkeydir, "rsa")
857        genSSHKey(sshkeydir, "dsa")   
858    except:
859        (type, value, tb) = sys.exc_info()
860        log_error("Failed to generate SSH keys for %s! - %s" % \
861                (host["host_name"], value))
862        # Rollback any changes
863        if commit == 1:
864            session.rollback()
865        return False
866
867    # Commit changeset if necessary
868    if commit==1:
869        session.commit()
870       
871    return True
872
873@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
874def createCfengineHostKeys(session_id, host_id):
875    """Creates a set of Cfengine keys for the specified host"""
876    from crcnetd.modules.ccs_host import ccs_host
877    session = getSessionE(session_id)
878   
879    host = ccs_host(session_id, host_id)
880   
881    # Check for existing changeset
882    commit = 0
883    if session.changeset == 0:
884        session.begin("Created CFengine keys for %s" % host["host_name"])
885        commit = 1
886   
887    try:
888        # Determine directories to store things in
889        workdir = session.revision.getWorkingDir()
890        cfkeydir = "%s/ppkeys" % (workdir)
891        ensureDirExists(cfkeydir)
892       
893        # Generate the keys
894        genCfKey(cfkeydir, host["ip_address"])
895    except:
896        (type, value, tb) = sys.exc_info()
897        log_error("Failed to generate CFengine keys for %s! - %s" % \
898                (host["host_name"], value))
899        # Rollback any changes
900        if commit == 1:
901            session.rollback()
902        return False
903
904    # Commit changeset if necessary
905    if commit==1:
906        session.commit()
907       
908    return True
909
910def genSSHKey(dir, type):
911    """Generates a SSH host key of the specified type
912
913    dir specifies the directory to place the key in
914    """
915    filename = "ssh_host_%s_key" % type
916    key = "%s/%s" % (dir, filename)
917    pubkey = "%s.pub" % key
918   
919    # Try and avoid needless recreating keys
920    if os.path.exists(key) and os.path.exists(pubkey):
921        log_debug("Skipping SSH key creation in %s. " \
922                "Keys already exist!" % dir)
923        return
924    try:
925        os.remove(key)
926        os.remove(pubkey)
927    except:
928        pass
929       
930    fh = os.popen("cd %s &>/dev/null && /usr/bin/ssh-keygen -q -f %s " \
931            "-N '' -t %s || echo \"unable to access %s\"" % \
932            (dir, filename, type, dir))
933    output = fh.readlines()
934    rv = fh.close()
935    if rv != None or (len(output)>0 and output[0].startswith("unable")):
936        raise ccs_cfengine_error("Could not create SSH %s host key: %s" % \
937                (type, "".join(output)))
938   
939def genCfKey(dir, host_ip):
940    """Generates a Cfengine host key pair
941
942    dir specifies the directory to place the key in
943    """
944    filename = "root-%s" % host_ip
945    key = "%s/%s.priv" % (dir, filename)
946    pubkey = "%s/%s.pub" % (dir, filename)
947
948    # Try and avoid needless recreating keys
949    if os.path.exists(key) and os.path.exists(pubkey):
950        log_debug("Skipping Cfengine key creation in %s. " \
951                "Keys already exist!" % dir)
952        return
953    try:
954        os.remove(key)
955        os.remove(pubkey)
956    except:
957        pass
958
959    fh = os.popen("cd %s &>/dev/null && /usr/sbin/cfkey -f %s || echo " \
960            "\"unable to access %s\"" % (dir, filename, dir))
961    output = fh.readlines()
962    rv = fh.close()
963    if rv != None or (len(output)>0 and output[0].startswith("unable")):
964        raise ccs_cfengine_error("Could not create CFengine host key: %s" % \
965                "".join(output))
966   
967def getActiveRevision(hostname=None):
968    """Returns the active revision for the cfengine input directory
969
970    If a hostname is specified it looks in the hosts/<hostname> directory
971    specifically
972    """
973    global _cfInputDir
974
975    if hostname is None:
976        dir = _cfInputDir
977    else:
978        dir = "%s/hosts/%s" % (_cfInputDir, hostname)
979
980    # Get the revision number of the checked out revision
981    try:
982        contents = open("%s/ccs-revision" % dir, "r").read()
983        return int(contents.strip())
984    except: 
985        log_debug("Could not read ccs-revision", sys.exc_info())
986        pass
987   
988    # No checked out revision
989    return -1
990
991def getGeneratedRevision(hostname=None):
992    """Returns the latest revision that has been generated
993
994    If a hostname is specified it looks in the hosts/<hostname> directory
995    specifically
996    """
997   
998    if hostname is None:
999        dir = ""
1000    else:
1001        dir = "inputs/hosts/%s" % hostname
1002   
1003    revision = ccs_revision(checkout=False)
1004    return revision.getYoungestRevision(dir)
1005
1006@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
1007def getCfRunLogs(session_id):
1008    """Returns the output of the last cfrun execution for all active hosts"""
1009    from crcnetd.modules.ccs_host import getHostList
1010    from crcnetd.modules.ccs_status import getHostStatus
1011    global _cfBaseDir, _cfrunStatus
1012   
1013    hosts = {}
1014   
1015    outputDir = "%s/outputs" % _cfBaseDir
1016    domain = config_get_required("network", "domain")
1017
1018    # Get a list of active hosts
1019    for host in getHostList(session_id):
1020        # Skip hosts that are disabled
1021        if not host["host_active"]:
1022            continue
1023        # Store enabled hosts
1024        hostname = host["host_name"]
1025        hosts[hostname] = host
1026        # Retrieve the information about it
1027        filename = "%s/%s.%s" % (outputDir, host["host_name"], domain)
1028        hosts[hostname]["last_run"] = -1
1029        hosts[hostname]["age"] = ""
1030        hosts[hostname]["age_rounded"] = ""
1031        hosts[hostname]["contents"] = []
1032        hosts[hostname]["no_lines"] = 0
1033        try:
1034            hostStatus = getHostStatus(hostname)
1035        except ccs_status_error:
1036            # Create a blank status entry
1037            host_id = hostname = getHostID(ADMIN_SESSION_ID, hostname)
1038            hostStatus = ccs_host_status(ADMIN_SESSION_ID, host_id)
1039        hosts[hostname]["active_rev"] = hostStatus.activeRevision
1040        hosts[hostname]["generated_rev"] = hostStatus.generatedRevision
1041        hosts[hostname]["operating_rev_status"] = \
1042                hostStatus.operatingRevisionStatus
1043        hosts[hostname]["operating_rev_text"] = \
1044                hostStatus.operatingRevisionText
1045        hosts[hostname]["operating_rev_hint"] = \
1046                hostStatus.operatingRevisionHint
1047        if hostStatus.infoUpdatedAt > 0:
1048            hosts[hostname]["operating_rev"] = int(hostStatus.operatingRevision)
1049            try:
1050                o = int(hosts[hostname]["operating_rev"])
1051                a = int(hosts[hostname]["active_rev"])
1052                g = int(hosts[hostname]["generated_rev"])
1053                if o > g:
1054                    hosts[hostname]["operating_rev_status"] = "critical"
1055                    hosts[hostname]["operating_rev_text"] = "Invalid Revision"
1056                    hosts[hostname]["operating_rev_hint"] = "The operating " \
1057                            "revision is greater than the newest generated " \
1058                            "revision!"
1059                elif o != a:
1060                    hosts[hostname]["operating_rev_status"] = "warning"
1061                    hosts[hostname]["operating_rev_text"] = ""
1062                    hosts[hostname]["operating_rev_hint"] = "The operating " \
1063                            "revision does not match the desired active " \
1064                            "revision!"                   
1065            except: pass
1066        else:
1067            hosts[hostname]["operating_rev"] = ""
1068        if hostname in _cfrunStatus.keys():
1069            hosts[hostname]["run_status"] = _cfrunStatus[hostname]["status"]
1070            hosts[hostname]["run_age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1071                    time.localtime(_cfrunStatus[hostname]["time"]))
1072            hosts[hostname]["run_age_rounded"] = roundTime(time.time() - \
1073                    _cfrunStatus[hostname]["time"])
1074        else:
1075            hosts[hostname]["run_status"] = ""
1076            hosts[hostname]["run_age"] = ""
1077            hosts[hostname]["run_age_rounded"] = ""
1078        hosts[hostname]["error"] = ""
1079        if not os.path.exists(filename):
1080            # No cfrun output for this host
1081            continue
1082        try:
1083            stat = os.stat("%s/%s.%s" % (outputDir, host["host_name"], domain))
1084            hosts[hostname]["age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1085                    time.localtime(stat[ST_MTIME]))
1086            hosts[hostname]["age_rounded"] = roundTime(time.time() - \
1087                    stat[ST_MTIME])
1088            hosts[hostname]["last_run"] = stat[ST_MTIME]
1089            hosts[hostname]["contents"] = open(filename).read()
1090            hosts[hostname]["no_lines"] = \
1091                    len(hosts[hostname]["contents"].split("\n"))
1092        except:
1093            (type, value, tb) = sys.exc_info()
1094            log_error("Failed to retrieve CfRun log for %s" % hostname, \
1095                    (type, value, tb))
1096            hosts[hostname]["error"] = value
1097            continue
1098
1099    # Return it
1100    return hosts
1101
1102@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
1103def pushConfig(session_id, hosts, params=[]):
1104    """Initiates a cfrun instance to push a new config out to a host"""
1105    global _threadLimit, _cfrunStatus, _cfrunLock
1106
1107    # PHP sends in an associative array we want the values only
1108    hosts = hosts.values()
1109
1110    # Loop through the specified hosts and mark them as pending
1111    _cfrunLock.acquire()
1112    try:
1113        try:
1114            for host in hosts:
1115                _cfrunStatus[host] = {"status":"pending","time":time.time()}
1116        except:
1117            log_error("Exception setting up config push!", sys.exc_info())
1118            raise ccs_cfengine_error("Could not initiate config push!")
1119    finally:
1120        _cfrunLock.release()
1121   
1122    # Start the threads
1123    initThread(startCfRunThreads, hosts, params)
1124    return True
1125
1126def startCfRunThreads(hosts, params):
1127
1128    # Loop through the hosts and start threads to do the cfrun stuff
1129    for host in hosts:
1130        _threadLimit.acquire()
1131        try:
1132            initThread(doCfrun, host, params)
1133        except:
1134            log_error("Failed to start cfrun instance for host: %s!" % \
1135                    host, sys.exc_info())
1136            _threadLimit.release()
1137    return True
1138
1139def doCfrun(host_name, params=[]):
1140    """Executes a cfrun instance to update the specified host
1141
1142    This function expects to be called in a thread and will take care of
1143    releasing the semaphore when it finishes.
1144    """
1145    global _threadLimit, _cfrunStatus, _cfrunLock, _cfBaseDir
1146   
1147    outputDir = "%s/outputs" % _cfBaseDir
1148    domain = config_get_required("network", "domain")
1149    cmdline = ""
1150    filename = ""
1151   
1152    # Make sure that any errors are captured so that we can't exit without
1153    # releasing the semaphore
1154    try:
1155        # Record that this host is being processed
1156        _cfrunLock.acquire()
1157        try:
1158            if host_name in _cfrunStatus.keys():
1159                if _cfrunStatus[host_name]["status"] != "pending":
1160                    # This host is already being processed!
1161                    _threadLimit.release()
1162                    return
1163            _cfrunStatus[host_name] = {"status":"running","time":time.time()}
1164        finally:
1165            _cfrunLock.release()
1166        # Ouput filename
1167        filename = "%s/%s.%s" % (outputDir, host_name, domain)
1168        try:
1169            os.unlink(filename)
1170        except:
1171            log_warn("Could not remove old logfile before cfrun! - %s" % \
1172                    filename, sys.exc_info())
1173        # Build the command line
1174        cmdline = "/usr/sbin/cfrun %s" % host_name
1175        if len(params) > 0:
1176            cmdline += " -- "
1177            for i,param in params.items(): cmdline += "%s " % param
1178        # Execute the command
1179        fp = os.popen(cmdline, "r")
1180        output = fp.read().strip()
1181        rv = fp.close()
1182        if rv is not None:
1183            code="failed"
1184        else:
1185            code="succeeded"
1186        # Retrieve the actual cfrun output
1187        try:
1188            contents = open(filename, "r").read()
1189        except:
1190            log_warn("Could not read logfile from cfrun! - %s" % filename, \
1191                    sys.exc_info())
1192            contents = ""
1193        # Prepend the output from the execution to the log
1194        fp = open(filename, "w")
1195        fp.write("%s\n" % ("*"*80))
1196        fp.write("Command execution %s!\n" % code)
1197        fp.write("Cmdline: %s\n" % cmdline)
1198        if len(output) > 0:
1199                fp.write("\n%s\n" % output)
1200        fp.write("%s\n\n" % ("*"*80))
1201        fp.write(contents)
1202        fp.close()
1203        # Record that this host is no longer being processed
1204        _cfrunLock.acquire()
1205        try:
1206            del _cfrunStatus[host_name]
1207        finally:
1208            _cfrunLock.release()
1209    except:
1210        (type, value, tb) = sys.exc_info()
1211        # Record that this host is no longer being processed
1212        _cfrunLock.acquire()
1213        try:
1214            del _cfrunStatus[host_name]
1215        finally:
1216            _cfrunLock.release()
1217        # Record an error in the log file
1218        if filename != "":
1219            try:
1220                try:
1221                    contents = open(filename, "r").read()
1222                except:
1223                    log_warn("Could not read logfile from cfrun! - %s" % \
1224                            filename, sys.exc_info())
1225                    contents = ""
1226                fp = open(filename, "w")
1227                fp.write("%s\n" % ("*"*80))
1228                fp.write("Command execution failed!\n")
1229                fp.write("Cmdline: %s\n" % cmdline)
1230                fp.write("Error: %s\n" % value)
1231                fp.write("%s\n\n" % ("*"*80))
1232                fp.write(contents)
1233                fp.close()
1234            except: pass
1235        log_error("Failed to initiate cfrun for host: %s" % host_name, \
1236                (type, value, tb))
1237   
1238    # Release the semaphore
1239    _threadLimit.release()
1240    return
1241
1242#####################################################################
1243# Functions to maintain / deal with checked out Cfengine configs
1244#####################################################################
1245@catchEvent("revisionCreated")
1246def updateCfInputDirCB(eventName, host_id, session_id, **kwargs):
1247    """Callback function to trigger updates when revisions are created"""
1248   
1249    autoupdate = int(config_get("cfengine", "autoupdate", "0"))
1250    if autoupdate:
1251        updateCfInputDir(session_id)
1252   
1253@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
1254def updateCfInputDir(session_id, revNum=-1):
1255    """Updates the cfengine input directory.
1256
1257    Triggered explicitly by user or on the revisionCreated hook.
1258    """
1259   
1260    svn = pysvn.Client()       
1261
1262    # See if there is a checked out revision
1263    gotrev = 0
1264    try:
1265        list = svn.ls(_cfInputDir)
1266        gotrev = 1
1267    except:
1268        # No checked out revision, remove bogus directory
1269        removeDir(_cfInputDir)
1270        ensureDirExists(_cfInputDir)
1271        pass
1272
1273    if revNum == -1:
1274        # Use head revision
1275        revH = pysvn.Revision(pysvn.opt_revision_kind.head)
1276    else:
1277        # Use specified revision
1278        revH = pysvn.Revision(pysvn.opt_revision_kind.number, revNum)
1279   
1280    if gotrev:
1281        # Update the directory
1282        rev = svn.update(_cfInputDir, True, revH)
1283    else:
1284        # Checkout the directory
1285        rev = svn.checkout("%s/inputs" % _svnroot, _cfInputDir, True, revH)
1286   
1287    # Update the revision information files
1288    log_command("%s/update-revisioninfo \"%s\"" % (_cfInputDir, _cfInputDir))
1289
1290    log_info("Updated cfengine configuration to revision %s." % rev.number)
1291
1292#####################################################################
1293# Functions to support the web based configuration browser
1294#####################################################################
1295@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
1296def viewConfigAtPath(session_id, path, revno=None, order=None, direction=0, \
1297        doMarkup=True):
1298    """Returns the configurations present at the specified path in the repo
1299   
1300    If the path is a directory you get a directory listing back
1301    If the path is a file you get the file contents back along with other
1302    metadata.
1303
1304    If the revno parameter is set the configuration is shown as it existed
1305    in the specified revision. If the revno parameter is None then the
1306    current (HEAD) revision is shown.
1307   
1308    The order parameter may be used to specify a parameter and order to sort
1309    the list that is returned. Valid parameters are name, size and date. Order
1310    may be 1 for descending or 0 for ascending.
1311   
1312    Unless the markup parameter is set to false the contents of the file will
1313    be processed through enscript and HTML will be returned.
1314    """
1315
1316    items = {}
1317   
1318    revision = ccs_revision(checkout=False)
1319    path = path.replace("//","/")
1320   
1321    (revno, entries) = revision.ls(path, revno)
1322    if revno == -1:
1323        # An error occured
1324        raise ccs_cfengine_error("Could not retrieve config at path: %s" % \
1325                path)
1326    for entry in entries:
1327        entry["path"] = entry["name"]
1328        entry["name"] = os.path.basename(entry["name"])
1329        if entry["path"].find("%s/" % entry["name"]) != -1:
1330            # psyvn bug, filename is duplicated
1331            entry["path"] = entry["path"][:-(len(entry["name"])+1)]
1332        if entry["path"].startswith(revision.svnroot):
1333            entry["path"] = entry["path"][len(revision.svnroot)+1:]
1334        rev = {}
1335        rev["date"] = entry["created_rev"].date
1336        rev["kind"] = entry["created_rev"].kind
1337        rev["number"] = entry["created_rev"].number
1338        rev["log"] = revision.getLog(rev["number"])
1339        entry["created_rev"] = rev
1340        entry["age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1341                time.localtime(entry["time"]))
1342        entry["age_rounded"] = roundTime(time.time() - entry["time"])
1343
1344    # Sort if necessary
1345    if order is not None and order != "":
1346        direction = int(direction)
1347        if order not in ["name", "size", "date"]:
1348            raise ccs_cfengine_error("Invalid order parameter: %s" % order)
1349        if direction != 0 and direction != 1:
1350            raise ccs_cfengine_error("Invalid direction parameter: %s" % \
1351                    direction)
1352        if order == "date": 
1353            order="time"
1354            direction += 1
1355        if direction == 1:
1356            entries.sort(lambda x,y: cmp(x[order], y[order])*-1)
1357        else:
1358            entries.sort(lambda x,y: cmp(x[order], y[order]))
1359
1360    # Return now if there are multiple files or no files
1361    if len(entries) == 0 or len(entries) > 1:
1362        return [revno, entries]
1363    # or a single directory
1364    if entries[0]["kind"] != pysvn.node_kind.file:
1365        return [revno, entries]
1366
1367    # Single file, does it match the path we were given?
1368    if entries[0]["path"] == path:
1369        # Get the contents
1370        file = revision.getFile(path)
1371        props = revision.getProps(path)
1372        entries[0]["props"] = props
1373        if doMarkup:
1374            if "ccs:format" in props.keys():
1375                entries[0]["contents"] = markup(file, props["ccs:format"])
1376            else:
1377                entries[0]["contents"] = markup(file)
1378
1379    return [revno, entries]
1380
1381def markup(file, format="sh"):
1382    """Passes the file through enscript for syntax highlighting"""
1383   
1384    # Generate the command line to pass to enscript
1385    cmdline = config_get("cfengine", "enscript_path", "enscript")
1386    cmdline += ' --color -h -q --language=html -p - -E' + format
1387    log_debug("Enscript command line: %s" % cmdline)
1388   
1389    # Run enscript
1390    (fdi, fdo) = os.popen2(cmdline)
1391    fdi.write(file)
1392    fdi.close()
1393    odata = fdo.read()
1394    rv = fdo.close()
1395    if rv is not None:
1396        log_warn("Could not run enscript to markup file!")
1397        log_debug(odata)
1398        return file.split("\n")
1399
1400    # Strip header and footer
1401    i = odata.find('<PRE>')
1402    beg = i > 0 and i + 6
1403    i = odata.rfind('</PRE>')
1404    end = i > 0 and i or len(odata)
1405    odata = EnscriptDeuglifier().format(odata[beg:end])
1406    return odata.splitlines()
1407
1408# Enscript Deuglifier code from Trac
1409#
1410# Copyright (C) 2003-2006 Edgewall Software
1411# All rights reserved.
1412#
1413# Licensed using the Modified BSD license
1414class EnscriptDeuglifier(object):
1415    def __new__(cls):
1416        self = object.__new__(cls)
1417        if not hasattr(cls, '_compiled_rules'):
1418            cls._compiled_rules = re.compile('(?:'+'|'.join(cls.rules())+')')
1419        self._compiled_rules = cls._compiled_rules
1420        return self
1421   
1422    def format(self, indata):
1423        return re.sub(self._compiled_rules, self.replace, indata)
1424
1425    def replace(self, fullmatch):
1426        for mtype, match in fullmatch.groupdict().items():
1427            if match:
1428                if mtype == 'font':
1429                    return '<span>'
1430                elif mtype == 'endfont':
1431                    return '</span>'
1432                return '<span class="code-%s">' % mtype
1433   
1434    def rules(cls):
1435        return [
1436            r'(?P<comment><FONT COLOR="#B22222">)',
1437            r'(?P<keyword><FONT COLOR="#5F9EA0">)',
1438            r'(?P<type><FONT COLOR="#228B22">)',
1439            r'(?P<string><FONT COLOR="#BC8F8F">)',
1440            r'(?P<func><FONT COLOR="#0000FF">)',
1441            r'(?P<prep><FONT COLOR="#B8860B">)',
1442            r'(?P<lang><FONT COLOR="#A020F0">)',
1443            r'(?P<var><FONT COLOR="#DA70D6">)',
1444            r'(?P<font><FONT.*?>)',
1445            r'(?P<endfont></FONT>)'
1446        ]
1447    rules = classmethod(rules)
1448
1449#####################################################################
1450# Cfengine Resource Server
1451#####################################################################
1452class cfserver(resource.Resource):
1453    """Implements a simple webserver to retrieve resources related to cfengine
1454
1455    Called when the main server receives a query for /cfengine/*
1456
1457    Currently supports the following resources
1458    /cfengine/update.conf/<host_name>
1459        Retrieves an update.conf file for the specified host
1460    /cfkey/keyname
1461        Retrieves the specified cfengine key
1462       
1463    Requests to this resource are restricted to the IP address ranges
1464    specified in the allow_requests_from configuration directive.
1465    """
1466   
1467    resourceName = "CFengine Resource Server"
1468
1469    # Mark as leaf so that render gets called
1470    isLeaf = 1
1471   
1472    def render(self, request):
1473
1474        # XXX: Verify request IP
1475       
1476        log_debug("Received cfengine request for %s" % request.path)
1477       
1478        parts = request.path.split("/")
1479        path = "/".join(parts[2:])
1480       
1481        # Process request
1482        if path.startswith("update.conf"):
1483            return self.getUpdateConf(path, request)
1484        elif path.startswith("cfkey"):
1485            return self.getCfkey(path, request)
1486        elif path.startswith("hostvars"):
1487            return self.getHostvars(path, request)
1488        else:
1489            request.setResponseCode(404, "Unknown cfengine resource requested!")
1490            request.finish()
1491            return server.NOT_DONE_YET
1492
1493    def getCfkey(self, path, request):
1494        """Returns the contents of a cfengine key for the specified host."""
1495       
1496        parts = path.split("/")
1497       
1498        # Get a revision to grab the key from
1499        revision = ccs_revision()
1500        keydir = "%s/ppkeys" % revision.getWorkingDir()
1501       
1502        # Retrieve the requested key
1503        keyfile = "%s/%s" % (keydir, parts[1])
1504
1505        try:
1506            fd = open(keyfile, "r")
1507            content = fd.readlines()
1508            fd.close()
1509        except:
1510            log_warn("Could not open Cfengine key file!", sys.exc_info())
1511            request.setResponseCode(404, "Unknown cfengine key requested!")
1512            request.finish()
1513            return server.NOT_DONE_YET
1514
1515        return "".join(content)
1516
1517    def getUpdateConf(self, path, request):
1518        """Returns the contents of an update.conf file for the specified host.
1519
1520        Currently the host parameter is ignored.
1521        """
1522       
1523        # Get a revision to grab the update.conf from
1524        revision = ccs_revision()
1525        revdir = revision.getConfigBase()
1526       
1527        try:
1528            fd = open("%s/cfconf/update.conf" % revdir, "r")
1529        except:
1530            request.setResponseCode(503, "update.conf unavailable. Please " \
1531                    "try again later")
1532            request.finish()
1533            return server.NOT_DONE_YET
1534       
1535        contents = fd.readlines()
1536        fd.close()
1537       
1538        return "".join(contents)
1539   
1540    def getHostvars(self, path, request):
1541        """Returns the contents of an update.conf file for the specified host.
1542
1543        Currently the host parameter is ignored.
1544        """
1545       
1546        parts = path.split("/")
1547        if len(parts)<2:
1548            request.setResponseCode(404, "Hostname not specified" % host)
1549            request.finish()
1550            return server.NOT_DONE_YET
1551        host = parts[1]
1552       
1553        # Get a revision to grab the hostvars file from
1554        revision = ccs_revision()
1555        revdir = revision.getConfigBase()
1556       
1557        try:
1558            fd = open("%s/hosts/%s/cf.hostvars" % (revdir, host), "r")
1559        except:
1560            request.setResponseCode(404, "%s hostvars not found!" % host)
1561            request.finish()
1562            return server.NOT_DONE_YET
1563       
1564        contents = fd.readlines()
1565        fd.close()
1566       
1567        return "".join(contents)
1568   
1569#####################################################################
1570# Revision Class
1571#####################################################################
1572class ccs_revision:
1573    """Wrapper for a revision of files.
1574   
1575    This class wraps the generation of an entire revision of the configuration
1576    files managed by this system.
1577   
1578    It contains the methods needed to generate the files, insert them into
1579    version control (svn) and then update then pass the resulting revision
1580    identifier back to it's caller.
1581    """
1582
1583    # Location of the svnroot
1584    svnroot = None
1585   
1586    def __init__(self, parentSession=None, changeset=None, checkout=True):
1587        """Creates a ccs_revision class.
1588
1589        If parentSession or changeset are not specified or None, a read-only
1590        revision is created. This is only useful if you want to inspect the
1591        repository without making any changes.
1592        """
1593       
1594        self.mParentSession = parentSession
1595        self.mChangeset = changeset
1596        self.checkout = checkout
1597        self.pendingProps = {}
1598        self.lock = threading.RLock()
1599       
1600        # Get a SVN client to use for this revision
1601        self.mSvn = pysvn.Client()
1602       
1603        if self.checkout:
1604            # Setup a working directory for this revision
1605            self.rDir = mkdtemp("", "ccsd")
1606       
1607            # Checkout the current configuration HEAD to this directory
1608            rev = self.mSvn.checkout(self.svnroot, self.rDir)
1609            self.mCurRev = rev
1610       
1611            # Check basic repository structure
1612            if self.mParentSession is not None and self.mChangeset is not None:
1613                self.checkRepoStructure()
1614        else:
1615            self.rDir = None
1616            self.mCurRev = None
1617           
1618        # Start with no errors
1619        self.mErrors = {}
1620       
1621    def __del__(self):
1622       
1623        # Nothing to check if nothing was checked out
1624        if self.rDir is None:
1625            return
1626
1627        # Don't check the status of a read-only revision
1628        if self.mParentSession is None or self.mChangeset is None:
1629            removeDir(self.rDir)
1630            return
1631       
1632        self.lock.acquire()
1633        try:
1634            # Check status of working directory
1635            try:
1636                status = self.mSvn.status(self.rDir, True, True)
1637            except pysvn.ClientError:
1638                # Probably never got checked out
1639                status = []           
1640        finally:
1641            self.lock.release()
1642           
1643        flag=0
1644        for entry in status:
1645            if entry.text_status == pysvn.wc_status_kind.ignored:
1646                continue
1647            if entry.text_status != pysvn.wc_status_kind.normal:
1648                log_debug("File %s modified (%s) in changeset %s" % \
1649                        (entry.path, entry.text_status, self.mChangeset))
1650                flag=1
1651       
1652        if flag==1:
1653            log_warn("Revision object for changeset %s in " \
1654                    "session %s deleted before checkin. Changes lost!" % \
1655                    (self.mChangeset, self.mParentSession.session_id))
1656               
1657        # Clean up the working directory
1658        removeDir(self.rDir)
1659       
1660    def getWorkingDir(self):
1661        """Returns the path that the repository is checked out into"""
1662
1663        return self.rDir
1664   
1665    def getConfigBase(self):
1666        """Returns the path that cfengine config files should live in"""
1667        if self.rDir is None:
1668            return None
1669        return "%s/inputs" % self.rDir
1670   
1671    def _checkForModified(self, cDir):
1672        """Performs svn actions on changed files in the specified directory"""
1673       
1674        # Nothing to check if no repository is checked out
1675        if self.rDir is None:
1676            return None
1677       
1678        # Don't check the status of a read-only revision
1679        if self.mParentSession is None or self.mChangeset is None:
1680            log_warn("Cannot check status on a read-only revision!")
1681            return
1682       
1683        # Recurse through if we were passed a list
1684        if type(cDir) == type([]):
1685            for d in cDir:
1686                self._checkForModified(d)
1687            return
1688
1689        self.lock.acquire()
1690        try:
1691            # Check state
1692            status = self.mSvn.status(cDir, True, True)
1693            for entry in status:
1694                # Ensure that ccs-revision files are always ignored
1695                if os.path.isdir(entry.path):
1696                    self.propset(entry.path, "svn:ignore", "ccs-revision")
1697                if entry.text_status == pysvn.wc_status_kind.unversioned:
1698                    if entry.path.startswith(self.rDir):
1699                        ename = entry.path[len(self.rDir)+1:]
1700                    log_debug("Added %s in changeset %s" % \
1701                            (ename, self.mChangeset))
1702                    self.mSvn.add(entry.path, False)
1703                    # Recurse if we added a directory
1704                    if os.path.isdir(entry.path):
1705                        self._checkForModified(entry.path)
1706                    else:
1707                        # Ensure the date property is set on files
1708                        self.propset(entry.path, "svn:keywords", "Date")
1709                        # Check if there are any other pending properties to
1710                        # set
1711                        path = entry.path
1712                        if path in self.pendingProps.keys():
1713                            for prop,value in self.pendingProps[path].items():
1714                                self.propset(path, prop, value, False)
1715                elif entry.text_status == pysvn.wc_status_kind.modified or \
1716                        entry.text_status == pysvn.wc_status_kind.added or \
1717                        entry.text_status == pysvn.wc_status_kind.ignored or \
1718                        entry.text_status == pysvn.wc_status_kind.deleted:
1719                    # Don't need to mention modified files that are already
1720                    # processed
1721                    pass
1722                elif entry.text_status != pysvn.wc_status_kind.normal:
1723                    log_debug("%s (%s) has bad state in changeset %s!" % \
1724                            (entry.path, entry.text_status, self.mChangeset))
1725        finally:
1726            self.lock.release()
1727       
1728    def propset(self, path, prop, value, canDefer=True):
1729        """Sets the specified property on the specified path"""
1730       
1731        if self.mParentSession is None or self.mChangeset is None:
1732            raise ccs_revision_error("Cannot set property on read-only " \
1733                    "revision")
1734       
1735        # Use the svn repo directly if nothing is checked out
1736        if self.rDir is None:
1737            base = self.svnroot
1738        else:
1739            base = self.rDir
1740               
1741        try:
1742            if not path.startswith(base):
1743                filename = "%s/%s" % (base, path)
1744            else:
1745                filename = path
1746            self.lock.acquire()
1747            try:
1748                rev = self.mSvn.propset(prop, value, filename)
1749            finally:
1750                self.lock.release()
1751            if self.rDir is None:
1752                self.saveRevProps(rev)
1753        except pysvn.ClientError:
1754            # Add to a list of pending properties that we'll try and set on
1755            # commit after adding the file. This covers the case where a
1756            # caller tries to set a property on an as yet unversioned file
1757            if filename in self.pendingProps.keys():
1758                self.pendingProps[filename][prop] = value
1759            else:
1760                self.pendingProps[filename] = {prop:value}
1761            log_debug("Deferred propset on %s" % filename, sys.exc_info())
1762        except:
1763            log_error("Could not set %s to %s on %s" % \
1764                    (prop, value, filename), sys.exc_info())
1765
1766    @registerEvent("revisionCreated")
1767    def checkin(self, message, paths=None):
1768        """Checks in the changes to the repository with the specified message"""
1769       
1770        if self.rDir is None:
1771            raise ccs_revision_error("Cannot checkin. Not checked out!")
1772       
1773        if self.mParentSession is None or self.mChangeset is None:
1774            raise ccs_revision_error("Cannot checkin a read-only revision")
1775       
1776        # Default to the whole repository
1777        if paths is None:
1778            paths = self.rDir
1779           
1780        # Check status of working directory and add / del files etc
1781        self._checkForModified(paths)
1782       
1783        self.lock.acquire()
1784        try:
1785            r = self.mSvn.checkin(paths, message, True)
1786            n = self.saveRevProps(r)
1787        finally:
1788            self.lock.release()
1789        if n<0:
1790            # Nothing changed
1791            return -1
1792       
1793        triggerEvent(self.mParentSession.session_id, "revisionCreated", \
1794                revision_no=n)
1795
1796        return n
1797
1798    def saveRevProps(self, r):
1799        """Saves customised properties against each revision
1800
1801        These properties are used to help keep track of how the database/svn
1802        repository changes match up.
1803        """
1804       
1805        if r is None:
1806            # Nothing changed
1807            return -1
1808
1809        # Set the author property on the checkin
1810        try:
1811            self.lock.acquire()
1812            try:
1813                r2 = self.mSvn.revpropset("svn:author", \
1814                        self.mParentSession.username, self.svnroot, r)
1815            finally:
1816                self.lock.release()
1817        except:
1818            (type, value, tb) = sys.exc_info()
1819            log_warn("Could not set author property on revision %s - %s" % \
1820                    (r.number, value))
1821            r2 = r
1822       
1823        # Record the changeset that triggered this revision
1824        try:
1825            self.lock.acquire()
1826            try:
1827                r3 = self.mSvn.revpropset("ccs:changeset", "%s" % \
1828                        self.mChangeset, self.svnroot, r2)
1829            finally:
1830                self.lock.release()
1831        except:
1832            (type, value, tb) = sys.exc_info()
1833            log_warn("Could not set changeset property on " \
1834                    "revision %s - %s" % (r2.number, value))
1835
1836        log_info("Committed revision %s to version control" % r2.number)
1837        return r2.number
1838   
1839    def fileExists(self, path):
1840        """Checks if the specified file exists in the repository"""
1841       
1842        # Use the svn repo directly if nothing is checked out
1843        if self.rDir is None:
1844            base = self.svnroot
1845        else:
1846            base = self.rDir
1847               
1848        try:
1849            if not path.startswith(base):
1850                filename = "%s/%s" % (base, path)
1851            else:
1852                filename = path
1853            self.lock.acquire()
1854            try:
1855                e = self.mSvn.ls(filename)
1856            finally:
1857                self.lock.release()
1858            if len(e) > 0:
1859                return True
1860        except:
1861            log_warn("Failed to check existance of file: %s" % filename)
1862            return False
1863       
1864        return False
1865   
1866    def ls(self, path, revno=None):
1867        """Returns a list of entries in the specified directory"""
1868       
1869        # Use the svn repo directly if nothing is checked out
1870        if self.rDir is None:
1871            base = self.svnroot
1872        else:
1873            base = self.rDir
1874               
1875        try:
1876            if not path.startswith(base):
1877                filename = "%s/%s" % (base, path)
1878            else:
1879                filename = path
1880               
1881            self.lock.acquire()
1882            try:
1883                if revno is None or revno=="":
1884                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
1885                else:
1886                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
1887                e = self.mSvn.ls(filename.rstrip("/"), rev)
1888            finally:
1889                self.lock.release()
1890            if len(e) > 0:
1891                if revno is None:
1892                    revno = self.getYoungestRevision()
1893                return (revno, e)
1894        except:
1895            log_warn("Could not list directory: %s" % filename, sys.exc_info())
1896       
1897        return (-1, [])
1898       
1899    def getYoungestRevision(self, path=""):
1900        """Returns the number of the youngest revision in the repository"""
1901        revno = -1
1902
1903        if path != "":
1904            path = "%s/%s" % (self.svnroot, path)
1905        else:
1906            path = self.svnroot
1907       
1908        try:
1909            self.lock.acquire()
1910            t = mkdtemp("", "ccsd")
1911            try:
1912                l = self.mSvn.log(path)
1913                for entry in l:
1914                    # Discard revisions which touched only the directory
1915                    r1 = pysvn.Revision(pysvn.opt_revision_kind.number, \
1916                            entry["revision"].number-1)
1917                    r2 = pysvn.Revision(pysvn.opt_revision_kind.number, \
1918                            entry["revision"].number)
1919                    try:
1920                        diff = self.mSvn.diff(t, path, r1, r2).strip()
1921                    except:
1922                        (type, value, tb) = sys.exc_info()
1923                        if str(value).find("was not found") != -1:
1924                            # Earlier revision not found, must be added
1925                            revno = entry["revision"].number
1926                            break
1927                    ignore = True
1928                    for line in diff.split("\n"):
1929                        if line.strip() == "Property changes on:":
1930                            continue
1931                        if line.startswith("Property changes on:"):
1932                            ignore = False
1933                            break
1934                        if line.startswith("Index:"):
1935                            ignore = False
1936                            break
1937                    if ignore:
1938                        continue
1939                    # Found a revision which touched inside the directory
1940                    revno = entry["revision"].number
1941                    break
1942            finally:
1943                self.lock.release()
1944                removeDir(t)
1945        except:
1946            log_warn("Could not determine youngest revision", sys.exc_info())
1947
1948        return revno
1949   
1950    def getLog(self, revno):
1951        """Returns the log message for the specified revision"""
1952
1953        try:
1954            self.lock.acquire()
1955            try:
1956                rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
1957                info = self.mSvn.revpropget("svn:log", self.svnroot, rev)
1958            finally:
1959                self.lock.release()
1960            return info[1].strip()
1961        except:
1962            log_warn("Could not retrieve log for revision %s" % revno, \
1963                    sys.exc_info())
1964
1965        return ""
1966   
1967    def getFile(self, path, revno=None):
1968        """Returns the contents of the file at the specified path and rev"""
1969       
1970        # Use the svn repo directly if nothing is checked out
1971        if self.rDir is None:
1972            base = self.svnroot
1973        else:
1974            base = self.rDir
1975               
1976        try:
1977            if not path.startswith(base):
1978                filename = "%s/%s" % (base, path)
1979            else:
1980                filename = path
1981            self.lock.acquire()
1982            try:
1983                if revno is None:
1984                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
1985                else:
1986                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
1987                contents = self.mSvn.cat(filename.rstrip("/"), rev)
1988            finally:
1989                self.lock.release()
1990            return contents
1991        except:
1992            log_warn("Could not retrieve file: %s" % filename, sys.exc_info())
1993       
1994        return ""
1995
1996    def getProps(self, path, revno=None):
1997        """Returns the properties set on the specified file"""
1998       
1999        # Use the svn repo directly if nothing is checked out
2000        if self.rDir is None:
2001            base = self.svnroot
2002        else:
2003            base = self.rDir
2004               
2005        try:
2006            if not path.startswith(base):
2007                filename = "%s/%s" % (base, path)
2008            else:
2009                filename = path
2010            self.lock.acquire()
2011            try:
2012                if revno is None:
2013                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
2014                else:
2015                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
2016                contents = self.mSvn.proplist(filename.rstrip("/"), rev)
2017            finally:
2018                self.lock.release()
2019            if len(contents) > 0:
2020                return contents[0][1]
2021        except:
2022            log_warn("Could not retrieve properties: %s" % \
2023                    filename, sys.exc_info())
2024       
2025        return ""
2026
2027    def checkRepoStructure(self):
2028        """Checks the repository has all the required base directories.
2029
2030        If a base directory is not present it is created. The base directories
2031        are:
2032        inputs/        Host/Service configuration files
2033
2034        Further hierarchy within each base directory is the responsibility of
2035        other modules.
2036        """
2037       
2038        if self.rDir is None:
2039            raise ccs_revision_error("Cannot check repository structure. " \
2040                    "Not checked out!")
2041       
2042        if self.mParentSession is None or self.mChangeset is None:
2043            log_warn("Cannot check repository structure on a read-only " \
2044                    "revision")
2045            return
2046       
2047        flag = 0
2048       
2049        configsDir = self.getConfigBase()
2050        n = ensureDirExists(configsDir)
2051        self.lock.acquire()
2052        try:
2053            if n > 0:
2054                # Schedule Additions
2055                bDir = configsDir
2056                while n>1:
2057                    bDir = os.path.dirname(configsDir)
2058                    n-=1
2059                log_info("Created configuration directory (%s) within " \
2060                        "repository" % configsDir)
2061                self.mSvn.add(bDir, True)
2062                flag = 1
2063           
2064            # Commit directory changes immediately
2065            if flag==1:
2066                r = self.mSvn.checkin(self.rDir, \
2067                        "checkRepoStructure created missing directories", True)
2068        finally:
2069            self.lock.release()
2070
2071        # Check for the script that updates the ccs-revision files
2072        filename = "%s/update-revisioninfo" % configsDir
2073        docreate=False
2074        doupdate=False
2075        if not os.path.exists(filename):
2076            docreate=True
2077        else:
2078            contents = file(filename, "r").read()
2079            if contents.find("svn info") == -1:
2080                doupdate = True
2081        if docreate or doupdate:
2082            self.lock.acquire()
2083            try:
2084                # Create the script
2085                fp = open(filename, "w")
2086                fp.write("""#!/bin/bash
2087echo "Updating version information"
2088for d in `find $1 -type d | grep -v ".svn" | xargs`; do
2089    version=`svn info $d/* | grep "Last Changed Rev" | awk '{print $4}' | sort -nr | head -n1`
2090    echo "  $d -> $version"
2091    echo "$version" > $d/ccs-revision
2092done
2093# vim:set sw=4 ts=4 sts=4 et:
2094""")
2095                fp.close()
2096                if docreate:
2097                    self.mSvn.add(filename, False)
2098                    self.mSvn.propset("svn:executable", "true", filename)
2099                    action = "Created"
2100                else:
2101                    action = "Updated"
2102                self.mSvn.checkin(filename, \
2103                        "%s update-revisioninfo script" % action)
2104            finally:
2105                self.lock.release()
2106
2107def validateRepository(svnroot):
2108    """Checks for the existance of a valid svn repository at svnroot"""
2109
2110    # Extract the path from the URL
2111    svnpath = svnroot[svnroot.find("://")+3:]
2112
2113    # Check directory exists
2114    if not os.path.exists(svnpath):
2115        log_debug("Specified repository path does not exist: %s" % svnpath)
2116        return False
2117   
2118    # Check it looks a bit like a subversion repository
2119    fp = open("%s/README.txt" % svnpath, "r")
2120    if not fp:
2121        log_debug("No README.txt inside repository: %s" % svnpath)
2122        return False
2123    tmp = fp.read().strip()
2124    fp.close()
2125    if not tmp.startswith("This is a Subversion repository"):
2126        log_debug("invalid README.txt in repository: %s" % svnpath)       
2127        return False
2128
2129    # Final, master check, can we check it out, instantiate a revision class
2130    revision = ccs_revision(True, True)
2131    del revision
2132
2133    # All OK
2134    return True
2135
2136def createRepository(svnroot):
2137    """Initialises a blank repository and configures it for use by ccsd"""
2138
2139    # Look for the svnadmin utility
2140    fd = os.popen("/usr/bin/which svnadmin 2>/dev/null")
2141    path = fd.read().strip()
2142    if fd.close() != None:
2143        log_error("Cannot find svnadmin utility.")
2144        return False
2145   
2146    # Check we can execute the svnadmin utilty
2147    if not os.access(path, os.X_OK):
2148        log_error("Cannot execute svnadmin utility.")
2149        return False
2150
2151    # Try and create the repository
2152    svnpath = svnroot[svnroot.find("://")+3:]
2153    fd = os.popen("%s create --fs-type fsfs %s 2>&1" % (path, svnpath))
2154    result = fd.read().strip()
2155    if fd.close() != None:
2156        log_error("svnadmin create failed. See tb log for more details.")
2157        log_tb(None, result)
2158        return False
2159   
2160    # Add a pre-revprop-change hook to allow the daemon to set author
2161    # revision properties
2162    hook = "%s/hooks/pre-revprop-change" % svnpath
2163    fd = open(hook, "w")
2164    if not fd:
2165        log_warning("Unable to setup pre-revprop-change hook in new " \
2166                "repository!")
2167    else:
2168        fd.write("""#!/bin/bash
2169
2170# PRE-REVPROP-CHANGE HOOK
2171
2172# Created by the CRCnet Configuration System Daemon on
2173# %s
2174
2175# Allow all changes at this time
2176exit 0
2177""" % time.ctime())
2178        fd.close()
2179        os.chmod(hook, 0755)
2180
2181    # Success
2182    log_info("Subversion repository created at %s" % svnpath)
2183    return True
2184
2185#####################################################################
2186# Cfengine Integration Initialisation
2187#####################################################################
2188# Base CFengine paths
2189_cfBaseDir = ""
2190_cfInputDir = ""
2191_svnroot = ""
2192# Global information about template generation
2193_hostTemplates = {}
2194_networkTemplates = {}
2195_templateDir = ""
2196_templateModDir = ""
2197_events = {}
2198_templateStats = {}
2199_statsLock = None
2200# Global information about cfrun instances
2201_cfrunStatus = {}
2202_cfrunLock = None
2203# Semaphore to control how many threads we initiate
2204_threadLimit = None
2205
2206def initCFengine():
2207    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
2208    global _events, _cfBaseDir, _cfInputDir, _svnroot, _statsLock, _threadLimit
2209    global _cfrunLock
2210
2211    # Retrieve template directory configuration
2212    _templateDir = config_get("cfengine", "template_dir")
2213    if _templateDir == None or _templateDir == "":
2214        log_fatal("Invalid template directory (%s)!" % _templateDir)
2215    _templateModDir = config_get("cfengine", "template_module_dir")
2216    if _templateModDir == None or _templateModDir == "":
2217        log_fatal("Invalid template module directory (%s)!" % _templateModDir)
2218
2219    # Retrieve cfengine configuration file directory configuration
2220    _cfBaseDir = config_get("cfengine", "cfbase_dir")
2221    if _cfBaseDir == None or _cfBaseDir == "":
2222        log_fatal("Invalid cfengine base directory (%s)!" % _cfBaseDir)
2223    _cfInputDir = "%s/inputs" % _cfBaseDir
2224
2225    # Decide where the svn root is
2226    _svnroot = config_get("cfengine", "config_svnroot")     
2227    if _svnroot == None or _svnroot == "":
2228        log_fatal("Configuration Data subversion repository not specified!")
2229    # Check that the svnroot is a valid svn URL
2230    if _svnroot.find("://")==-1:
2231        log_fatal("Invalid svnroot URL: %s" % _svnroot)   
2232    # Let the revision class know where to get its data from
2233    ccs_revision.svnroot = _svnroot
2234    # Validate the repository and create it if it doesn't exist
2235    if not validateRepository(_svnroot):
2236        log_warn("No svn repository at specified path. Attempting " \
2237                "to create one.")
2238        if not createRepository(_svnroot):
2239            log_fatal("Could not create repository at %s" % _svnroot)
2240   
2241    # Initialise the statistics lock
2242    _statsLock = threading.Lock()
2243
2244    # Initialise the cfrun lock
2245    _cfrunLock = threading.Lock()
2246   
2247    # How many threads are we allowed to run at a time
2248    maxThreads = config_get("cfengine", "max_threads", DEFAULT_MAX_THREADS)
2249    suggestThreadpoolSize(maxThreads)
2250    _threadLimit = threading.Semaphore(maxThreads)
2251   
2252    # Initialise the templates
2253    initTemplates()
2254   
2255    # Register Cfengine Resource Server
2256    registerResource("cfengine", cfserver)
Note: See TracBrowser for help on using the repository browser.