Gitkrakenのタグを201503に移動させてlibフォルダを見ると
main.c core.c io.c queue.cがはいており
$ makeでa.outファイルができた。
$ ./a.outで入力待機状態になったようなのでstreemと
入力するとSTREEMと標準出力に表示された。
停止方法はないようなのでCTLR+Cで停止
main.cが全体の初期化と実行
core.cがメインループの制御(strm_loop関数）
io.cが標準入力、出力の制御、I/Oループの制御（io_loop関数）
queue.cがキューの制御(strm_queue_push関数がイベントを積み
strm_queue_exec関数がイベントを降ろして実行）
Ubuntu16.04LTS(64bit)にGitクライアントのGitKrakenをinstallした。
GitKrakenに
https://github.com/matz/streemをクローンさせて手元にリポジトリを作る
最初のタグ201502に移動してResr master to this commit>Hard-discard all changesを実行させると
リポジトリ内フォルダ(streem/src)に「Makefile lex.l parse.y」ができている。
$ sudo apt install flex
でflexをインストールしてmakeとコマンドを入力すると
yacc parse.y
flex lex.l
gcc -g - c y.tab.c -o parse.o
gcc parse.o -o streem
と実行され、新たにlex.yy.c parse.o y.tab.c streemファイルが作成されていた。
この構造体のメンバーには接頭子tp_が付けられている。
typeobjectの頭文字と予想される。
typedef struct _typeobject {
PyObject_VAR_HEAD
const char *tp_name; /* For printing, in format "<module>.<name>" */ ＃この構造体の名前を要求されたときのフォーマット？
Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ ＃メモリの上限値？
/* Methods to implement standard operations */
destructor tp_dealloc; #デストラクタ?
printfunc tp_print; #プリントファンクション？
getattrfunc tp_getattr; #この構造体のゲッター？
setattrfunc tp_setattr; ＃この構造体のセッタ？
PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
or tp_reserved (Python 3) */ ＃？
reprfunc tp_repr; #？
/* Method suites for standard classes */
PyNumberMethods *tp_as_number; ＃PyObjectが数値だった場合のメソッド(nb_add,nb_subtract,nb_multiply,nb_divmodなどへの関数へのポインタが並んで知る接頭子のnbは?
PySequenceMethods *tp_as_sequence; #PyObjectがシーケンスタイプのオブジエクトだった場合の構造体。接頭子のsqはシーケンス（sequence）由来と思われる。おそらくリスト、タプルに適用されるのだろう
PyMappingMethods *tp_as_mapping; ＃PyObjectがマッピングタイプのオブジエクトだった場合の構造体 辞書に適用されるのだろう。
/* More standard operations (here for binary compatibility) */
hashfunc tp_hash; ＃？
ternaryfunc tp_call; ＃？
reprfunc tp_str; ＃？
getattrofunc tp_getattro; ＃？
setattrofunc tp_setattro; ＃？
/* Functions to access object as input/output buffer */
PyBufferProcs *tp_as_buffer; ＃何かのバッフア？
/* Flags to define presence of optional/expanded features */
unsigned long tp_flags; ＃オプション機能/拡張機能の有無を定義するフラグ？
const char *tp_doc; /* Documentation string */ ＃ドキュメント文字列へのポインタ？
/* Assigned meaning in release 2.0 */
/* call function for all accessible objects */
traverseproc tp_traverse; ＃？
/* delete references to contained objects */
inquiry tp_clear; ＃？
/* Assigned meaning in release 2.1 */
/* rich comparisons */
richcmpfunc tp_richcompare; ＃？
/* weak reference enabler */
Py_ssize_t tp_weaklistoffset; ＃弱い参照？
/* Iterators */
getiterfunc tp_iter;
iternextfunc tp_iternext;
/* Attribute descriptor and subclassing stuff */
struct PyMethodDef *tp_methods; ＃Pythonビルトイン関数/メソッドへのポインタ
struct PyMemberDef *tp_members; ＃？
struct PyGetSetDef *tp_getset; ＃？
struct _typeobject *tp_base; ＃？
PyObject *tp_dict; ＃？
descrgetfunc tp_descr_get; ＃？
descrsetfunc tp_descr_set; ＃？
Py_ssize_t tp_dictoffset; ＃？
initproc tp_init; ＃？
allocfunc tp_alloc; ＃？
newfunc tp_new; ＃？
freefunc tp_free; /* Low-level free-memory routine */ ＃？
inquiry tp_is_gc; /* For PyObject_IS_GC */ ＃ガ－ベージコレクション？
PyObject *tp_bases; ＃？
PyObject *tp_mro; /* method resolution order */ ＃？
PyObject *tp_cache; ＃？
PyObject *tp_subclasses; ＃？
PyObject *tp_weaklist; ＃？
destructor tp_del; ＃？
/* Type attribute cache version tag. Added in version 2.6 */
unsigned int tp_version_tag; ＃？
destructor tp_finalize; ＃？
#ifdef COUNT_ALLOCS ＃なし
/* these must be last and never explicitly initialized */
Py_ssize_t tp_allocs;
Py_ssize_t tp_frees;
Py_ssize_t tp_maxalloc;
struct _typeobject *tp_prev;
struct _typeobject *tp_next;
#endif
} PyTypeObject;
構造体の先頭にあるPyObject_VAR_HEADはPyVarObjectを表すマクロ
PyVarObjectはPyObjectにPy_ssize_t ob_sizeを追加して拡張したもののようである。
もっとも頻度の高いキーワードPyObjectは Includeフォルダのobject.h内に定義してある
構造体でした。
Pythonの全てのオブジエクトの元になっている構造体である。
マクロを展開すると
typedef struct _object {
struct _object *_ob_next;
struct _object *_ob_prev;
Py_ssize_t ob_refcnt; #longかintの別定義
struct _typeobject *ob_type; typestruct.hに定義してある構造体
} PyObject;
_typeobject の定義
typedef struct _typeobject {
PyObject_VAR_HEAD
const char *tp_name; /* For printing, in format "<module>.<name>" */
Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
/* Methods to implement standard operations */
destructor tp_dealloc;
printfunc tp_print;
getattrfunc tp_getattr;
setattrfunc tp_setattr;
PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
or tp_reserved (Python 3) */
reprfunc tp_repr;
/* Method suites for standard classes */
PyNumberMethods *tp_as_number;
PySequenceMethods *tp_as_sequence;
PyMappingMethods *tp_as_mapping;
/* More standard operations (here for binary compatibility) */
hashfunc tp_hash;
ternaryfunc tp_call;
reprfunc tp_str;
getattrofunc tp_getattro;
setattrofunc tp_setattro;
/* Functions to access object as input/output buffer */
PyBufferProcs *tp_as_buffer;
/* Flags to define presence of optional/expanded features */
unsigned long tp_flags;
const char *tp_doc; /* Documentation string */
/* call function for all accessible objects */
traverseproc tp_traverse;
/* delete references to contained objects */
inquiry tp_clear;
/* rich comparisons */
richcmpfunc tp_richcompare;
/* weak reference enabler */
Py_ssize_t tp_weaklistoffset;
/* Iterators */
getiterfunc tp_iter;
iternextfunc tp_iternext;
/* Attribute descriptor and subclassing stuff */
struct PyMethodDef *tp_methods;
struct PyMemberDef *tp_members;
struct PyGetSetDef *tp_getset;
struct _typeobject *tp_base;
PyObject *tp_dict;
descrgetfunc tp_descr_get;
descrsetfunc tp_descr_set;
Py_ssize_t tp_dictoffset;
initproc tp_init;
allocfunc tp_alloc;
newfunc tp_new;
freefunc tp_free; /* Low-level free-memory routine */
inquiry tp_is_gc; /* For PyObject_IS_GC */
PyObject *tp_bases;
PyObject *tp_mro; /* method resolution order */
PyObject *tp_cache;
PyObject *tp_subclasses;
PyObject *tp_weaklist;
destructor tp_del;
/* Type attribute cache version tag. Added in version 2.6 */
unsigned int tp_version_tag;
destructor tp_finalize;
} PyTypeObject;
次はPyTypeObjectを調べよう
def iter_docs(file):
"""
指定したCファイルにインクルードされているヘッダファイルを返す
"""
for line in file:
if line.startswith("#include"):
#include文に続くヘッダファイル名を抜き出す
m = re.search(r'\".*\"',line)
if m != None:
yield m.group().strip("\"").split("/")[-1]
def get_hlist(path_name):
"""
指定されたCファイルのpath_nameをもらってそのCファイルにインクルード
されているヘッダファイルファイルをリストにして返す。
"""
h_file_list = []
for path in path_name:
with open(path) as f:
for hf in iter_docs(f):
h_file_list.append(hf)
h_file_list = list(set(h_file_list))
print("--head file---------------")
print("上位10位まで表示")
for v in h_file_list[:10]:
print(v)
print("len=",len(h_file_list))
return h_file_list
<結果>
最初10行まで表示
sha256module.c.h
unicodedefs.h
mappings_hk.h
ucs4lib.h
datetime.h
code.h
grammar.h
rotatingtree.h
fileobject.h
float.h
def freqdist_count(path_name):
"""
指定したpath_nameを読み込んで、そのテキスト内単語の頻度を表示する
"""
count_dic = {}
for path in path_name[:]:
with open(path,"r") as f:
text = f.read()
#コメントを削除（複数行も削除）
text = re.sub(r"/\*([^*]|\*[^/])*\*/","",text)
#{} ()を空白に置き換える
text = re.sub(r"\("," ",text)
text = re.sub(r"\)"," ",text)
text = re.sub(r"{"," ",text)
text = re.sub(r"}"," ",text)
#単独の数値を空白に置き換える
text = re.sub(r"[0-9]+"," ", text)
#カンマを空白に置き換える
text = re.sub(r"\,"," ",text)
#;を空白に置き換える
text = re.sub(r";"," ",text)
wList = text.split()
for w in wList:
if w in c_lang_keyword:
continue
count_dic[w] = count_dic.get(w,0) + 1
count_list= sorted(count_dic.items(), key=lambda x: x[1],reverse=True)
print("--freqdist---------------")
print("上位10位まで表示")
s = 0
for k,v in count_list[:10]:
print(k,":",v)
s += v
print("{:,}".format(s)
path_name = count_file_func(c_file_list)
freqdist_count(path_name)
<結果>
上位10位まで表示
PyObject : 12235
\ : 3648
Py_DECREF : 3485
Py_ssize_t : 3052
self : 3048
c : 2699
res : 2572
len : 1922
" : 1815
value : 1693
36,169
最頻度はPyObjectで、これが最重要データ構造と思われる。
179のファイルがどのフォルダーに分布しているのか調べた。
import glob
import os
c_file_list = ["_bisectmodule.c","_codecsmodule.c","_collectionsmodule.c","_csv.c",
"_functoolsmodule.c","_heapqmodule.c","_json.c","_localemodule.c",
"_lsprof.c","_math.c","_pickle.c","_randommodule.c","_sre.c","_stat.c",
"_struct.c","_weakref.c","arraymodule.c","atexitmodule.c","audioop.c",
"binascii.c","cmathmodule.c","_datetimemodule.c","errnomodule.c",
"faulthandler.c","gcmodule.c","hashtable.c","itertoolsmodule.c",
"main.c","mathmodule.c","md5module.c","mmapmodule.c","_opcode.c",
"_operator.c","parsermodule.c","posixmodule.c","rotatingtree.c",
"sha1module.c","sha256module.c","sha512module.c","signalmodule.c",
"symtablemodule.c","_threadmodule.c","_tracemalloc.c","timemodule.c",
"xxsubtype.c","zipimport.c","zlibmodule.c","fileio.c","bytesio.c",
"stringio.c","bufferedio.c","iobase.c","textio.c","_iomodule.c","adler32.c",
"compress.c","crc32.c","deflate.c","infback.c","inffast.c","inflate.c","inftrees.c",
"trees.c","uncompr.c","zutil.c","_codecs_cn.c","_codecs_hk.c","_codecs_iso2022.c",
"_codecs_jp.c","_codecs_kr.c","_codecs_tw.c","multibytecodec.c","_winapi.c",
"abstract.c","accu.c","boolobject.c","bytes_methods.c","bytearrayobject.c",
"bytesobject.c","capsule.c","cellobject.c","classobject.c","codeobject.c",
"complexobject.c","descrobject.c","dictobject.c","enumobject.c","exceptions.c",
"fileobject.c","floatobject.c","frameobject.c","funcobject.c","genobject.c",
"iterobject.c","listobject.c","longobject.c","memoryobject.c","methodobject.c",
"moduleobject.c","namespaceobject.c","object.c","obmalloc.c","odictobject.c",
"rangeobject.c","setobject.c","sliceobject.c","structseq.c","tupleobject.c",
"typeobject.c","unicodectype.c","unicodeobject.c","weakrefobject.c",
"acceler.c","bitset.c","firstsets.c","grammar.c","grammar1.c","listnode.c",
"metagrammar.c","myreadline.c","node.c","parser.c","parsetok.c","tokenizer.c",
"invalid_parameter_handler.c","winreg.c","config.c","getpathp.c","msvcrtmodule.c",
"pyhash.c","random.c","_warnings.c","asdl.c","ast.c","bltinmodule.c","ceval.c",
"codecs.c","compile.c","dynamic_annotations.c","dynload_win.c","errors.c",
"fileutils.c","formatter_unicode.c","frozen.c","future.c","getargs.c",
"getcompiler.c","getcopyright.c","getopt.c","getplatform.c","getversion.c",
"graminit.c","import.c","importdl.c","marshal.c","modsupport.c","mysnprintf.c",
"mystrtoul.c","peephole.c","pyarena.c","pyctype.c","pyfpe.c","pylifecycle.c",
"pymath.c","pytime.c","pystate.c","pystrcmp.c","pystrhex.c","pystrtod.c","dtoa.c",
"Python-ast.c","pythonrun.c","structmember.c","symtable.c","sysmodule.c",
"thread.c","traceback.c","dl_nt.c","getbuildinfo.c"]
path_name = []
file_name = []
dir_name = []
dir_count = dict(zip(['Modules\\zlib', 'Parser', 'Modules\\_io', 'Python', 'Objects', 'Modules', 'Modules\\cjkcodecs', 'PC'],[0,0,0,0,0,0,0,0]))
for p in glob.glob("**",recursive =True):
if p.split(".")[-1] != "c":
continue
base_name = os.path.basename(p)
parent_name = os.path.split(p)[0]
if base_name in c_file_list:
path_name.append(p)
dir_name.append(parent_name)
file_name.append(base_name)
dir_count[parent_name] += 1
#targetとなっているファイルが所属するデリクトリの集約とそのデリクトリに所属するファイル数の計上
for key in dir_count:
print(key,dir_count[key])
<結果>
Objects 39
Modules\_io 7
Parser 12
Modules\zlib 11
Modules\cjkcodecs 7
Python 48
PC 6
Modules 49
Modules フォルダーが一番多く、次がPythonフォルダーが続く。
