commit 70618b6b910904af006b41a75d60af8cf78f6b05
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2022-08-02T13:35:37Z |
|---|
| subject | Better caching mechanics |
|---|
commit 70618b6b910904af006b41a75d60af8cf78f6b05
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2022-08-02T13:35:37Z
Better caching mechanics
---
botanjs/classmap.py | 18 ++++++++++++-----
botanjs/compressor/closure.py | 9 +++++++--
botanjs/dummy.py | 8 +++++++-
botanjs/service/jclassresv.py | 44 +++++++++++++++-------------------------
botanjs/service/jwork.py | 4 ++--
botanjs/utils.py | 12 +++++++++++
tests.py | 47 +++++++++++++++++++++++++++++++++++++++++++
7 files changed, 104 insertions(+), 38 deletions(-)
diff --git a/botanjs/classmap.py b/botanjs/classmap.py
index bb9b386..2902d7f 100644
--- a/botanjs/classmap.py
+++ b/botanjs/classmap.py
@@ -1,11 +1,10 @@
#!/usr/bin/env python3
-import os;
-import re;
-import sys;
+import os, re, sys
from xml.dom import minidom
from collections import defaultdict
+from botanjs.utils import checksum
RegEx_N = re.compile( r"""
.*
@@ -136,7 +135,7 @@ class ClassMap:
return True;
- def drawMap( self, ns, ci, ce, cf ):
+ def drawMap( self, ns, ci, ce, cf, chksum ):
nsNode = self.getNode( ns )
# Source every:
@@ -150,6 +149,8 @@ class ClassMap:
if not srcEvery:
nsNode.setAttribute( "src", cf )
+ nsNode.setAttribute( "js", chksum["js"] )
+ nsNode.setAttribute( "css", chksum["css"] )
for ex in ce:
_t = eDef[ ex[0] ]
@@ -164,6 +165,8 @@ class ClassMap:
cNode.appendChild( impNode )
cNode.setAttribute( "src", cf )
+ cNode.setAttribute( "js", chksum["js"] )
+ cNode.setAttribute( "css", chksum["css"] )
# the file dose not export classes
# Hence it import for itself
@@ -187,7 +190,12 @@ class ClassMap:
continue
ns, ci, ce = classMeta( classFile )
+
+ chksum = {}
+ chksum[ "js" ] = checksum( classFile )
+ chksum[ "css" ] = checksum( classFile[:-2] + "css" )
+
classFile = classFile.replace( self.R + os.path.sep, "" )
- self.drawMap( ns, ci, ce, classFile )
+ self.drawMap( ns, ci, ce, classFile, chksum )
return self.DOM.toxml()
diff --git a/botanjs/compressor/closure.py b/botanjs/compressor/closure.py
index dc943cd..72d08c5 100644
--- a/botanjs/compressor/closure.py
+++ b/botanjs/compressor/closure.py
@@ -4,11 +4,11 @@ import os
from sys import platform
from tempfile import NamedTemporaryFile
from botanjs.config import Config as config
+from botanjs.service.jwork import log
COMPILER = config[ "BotanJS" ][ "ClosureCompiler" ]
-if not os.path.isfile( COMPILER ):
- raise Exception( "Compiler not found" )
+AVAILABLE = os.path.isfile( COMPILER )
COMPILER_OPTIONS = [
@@ -42,6 +42,11 @@ class Wrapper:
break
def compress( self, loc ):
+
+ if not AVAILABLE:
+ log.error( "Compiler not found" )
+ return
+
content = ""
with open( loc, "rb" ) as f:
content = f.read()
diff --git a/botanjs/dummy.py b/botanjs/dummy.py
index 9c2c9dc..990ebc2 100644
--- a/botanjs/dummy.py
+++ b/botanjs/dummy.py
@@ -1,7 +1,13 @@
#!/usr/bin/env python3
class log:
- def info( self, *args ):
+
+ @staticmethod
+ def info( *args ):
+ print( *args )
+
+ @staticmethod
+ def error( *args ):
print( *args )
class dummyTask( object ):
diff --git a/botanjs/service/jclassresv.py b/botanjs/service/jclassresv.py
index 0bca229..4f542f5 100644
--- a/botanjs/service/jclassresv.py
+++ b/botanjs/service/jclassresv.py
@@ -30,7 +30,11 @@ class Resolver:
def resource( self, elem ):
if "src" in elem.attrib:
- return elem.attrib[ "src" ]
+ return {
+ "src": elem.attrib[ "src" ]
+ , "js": elem.attrib[ "js" ]
+ , "css": elem.attrib[ "css" ]
+ }
parent = self.parentMap[ elem ]
@@ -153,9 +157,10 @@ class BotanClassResolver:
if src not in classFiles:
classFiles.append( src )
- def cssLookup( self, jsList, cssList ):
+ def cssLookup( self, classList, cssList ):
- for f in jsList:
+ for cdef in classList:
+ f = cdef[ "src" ]
possibleList = []
cssFile = os.path.splitext( f )[0] + ".css"
@@ -183,7 +188,7 @@ class BotanClassResolver:
if self.CR == None:
return None
- md5 = hashlib.md5( bytearray( "".join( fileList ), "utf-8" ) ).hexdigest()
+ md5 = hashlib.md5( bytearray( "|".join( x[mode] for x in fileList ), "utf-8" ) ).hexdigest()
cName[0] = oFHash = md5 + "." + mode
cFHash = md5 + ".c." + mode
@@ -193,19 +198,10 @@ class BotanClassResolver:
# Compressed file
cFile = os.path.join( self.CR, cFHash )
- dates = list(
- os.path.getmtime( os.path.join( self.R, x ) )
- if os.path.exists( os.path.join( self.R, x ) ) else -1
- for x in fileList
- )
-
- # Root file date
- dates.append( os.path.getmtime( os.path.join( self.R, "_this.js" ) ) );
-
- if self.flagCompress and self.useCache( cFile, dates ):
+ if self.flagCompress and self.useCache( cFile ):
return self.BotanCache( cFile, cFHash, self.returnDynamic )
- elif self.useCache( oFile, dates ):
+ elif self.useCache( oFile ):
self.JWork.saveCache(
oFile
# Content is None to initiate a compression
@@ -216,17 +212,8 @@ class BotanClassResolver:
return self.BotanCache( oFile, oFHash, False )
- def useCache( self, f, dList ):
- if not os.path.exists( f ):
- return False
-
- t = os.path.getmtime( f )
-
- for i in dList:
- if t < i:
- return False
-
- return True
+ def useCache( self, f ):
+ return os.path.exists( f )
def compileJs( self, cList, xList ):
md5 = [ None ]
@@ -244,6 +231,7 @@ class BotanClassResolver:
for f in cList:
+ f = f[ "src" ]
path = (
os.path.splitext( f )[0]
.replace( PY_SEP, "." )
@@ -285,8 +273,8 @@ class BotanClassResolver:
if cacheFile != None:
return cacheFile;
- # The root file
- outputCss = self.BotanFile( "_this.css" )
+ struct = "/* @ */"
+ outputCss = struct + self.BotanFile( "_this.css" )
for f in self.cleanList( cList ):
outputCss += self.BotanFile( f )
diff --git a/botanjs/service/jwork.py b/botanjs/service/jwork.py
index 2796059..a0aacc8 100644
--- a/botanjs/service/jwork.py
+++ b/botanjs/service/jwork.py
@@ -1,7 +1,5 @@
#!/usr/bin/env python3
import os
-from botanjs.compressor.closure import Wrapper as ClosureWrapper
-from botanjs.compressor.yui import Wrapper as YUIWrapper
from botanjs.classmap import ClassMap
CeleryExists = True
@@ -38,6 +36,7 @@ class JWork:
@app.task()
def compressJs( md5, externs ):
+ from botanjs.compressor.closure import Wrapper as ClosureWrapper
log.info( "Compress js: " + md5 )
w = ClosureWrapper()
w.scanExterns( externs )
@@ -45,6 +44,7 @@ class JWork:
@app.task()
def compressCss( md5 ):
+ from botanjs.compressor.yui import Wrapper as YUIWrapper
log.info( "Compress css: " + md5 )
w = YUIWrapper()
w.compress( md5 )
diff --git a/botanjs/utils.py b/botanjs/utils.py
new file mode 100644
index 0000000..07bef39
--- /dev/null
+++ b/botanjs/utils.py
@@ -0,0 +1,12 @@
+from functools import lru_cache
+from zlib import adler32 as _HashFunc
+HashFunc = lambda v: hex( _HashFunc( v ) )[2:]
+
+
+@lru_cache( maxsize = 1024 )
+def checksum( file_path ):
+ try:
+ with open( file_path, "rb" ) as f:
+ return HashFunc( f.read() )
+ except FileNotFoundError:
+ return HashFunc( b"" )
diff --git a/tests.py b/tests.py
new file mode 100644
index 0000000..c7cd4eb
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,47 @@
+#!env/bin/python
+import os, sys
+sys.path.append( os.path.abspath( "." ) )
+
+from botanjs.service.jwork import app, JWork
+from botanjs.config import Config as config
+
+SiteRoot = os.path.abspath( "." )
+
+# Setting the SiteRoot for config
+config["Paths"]["SiteRoot"] = SiteRoot
+
+jsCache = config["Paths"]["Cache"]
+jsRoot = config["BotanJS"]["SrcDir"]
+
+bmap = os.path.join( jsCache, "botanjs", "bmap.xml" )
+
+app.conf.update( broker_url = config["BotanJS"]["CeleryBroker"] )
+
+JWork.buildClassMap.delay( jsRoot, bmap )
+
+from botanjs.service.jclassresv import BotanClassResolver as JCResv
+srvHandler = JCResv( JWork, jsRoot, bmap, jsCache )
+
+import unittest
+
+class TestStringMethods( unittest.TestCase ):
+
+ # Run each twice to test the cache capabilities
+ def test_ojscall( self ):
+ for _ in range(0,2):
+ s = srvHandler.getAPI( "System", mode = "rjs" )
+ self.assertTrue( "BotanJS.define( \"System\" );" in s )
+
+ def test_import( self ):
+ for _ in range(0,2):
+ s = srvHandler.getAPI( "System.Policy", mode = "rjs" )
+ self.assertTrue( "BotanJS.define( \"System.Policy\" );" in s )
+ self.assertTrue( "BotanJS.define( \"System.Global\" );" in s )
+
+ def test_cssInheritance( self ):
+ for _ in range(0,2):
+ s = srvHandler.getAPI( "System", mode = "rcss" )
+ self.assertTrue( "/* @ */" in s )
+
+if __name__ == '__main__':
+ unittest.main()