Skip to content

Instantly share code, notes, and snippets.

@goropikari

goropikari/julia_array_implement.ipynb

Last active March 22, 2019 05:26
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save goropikari/216bda6b0d3fb339a238eb99179c5722 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save goropikari/216bda6b0d3fb339a238eb99179c5722 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
julia_array_implement.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Copyright (c) 2019 goropikari \n",
"Released under the MIT license \n",
"https://opensource.org/licenses/mit-license.php"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Julia Version 1.1.0\n",
"Commit 80516ca202 (2019-01-21 21:24 UTC)\n",
"Platform Info:\n",
" OS: Linux (x86_64-pc-linux-gnu)\n",
" CPU: Intel(R) Core(TM) i5-4460T CPU @ 1.90GHz\n",
" WORD_SIZE: 64\n",
" LIBM: libopenlibm\n",
" LLVM: libLLVM-6.0.1 (ORCJIT, haswell)\n",
"Environment:\n",
" JULIA_SHELL = /bin/bash\n",
" JULIA_EDITOR = nvim\n"
]
}
],
"source": [
"versioninfo()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"using Libdl"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"arraycap (generic function with 1 method)"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# derive the capacity of a given array\n",
"header = if ispath(\"/usr/include/julia/julia.h\")\n",
" \"/usr/include/julia\"\n",
"elseif ispath(\"$(dirname(Base.julia_cmd()[1]))/../include/julia/julia.h\")\n",
" \"$(dirname(Base.julia_cmd()[1]))/../include/julia\"\n",
"else\n",
" println(\"Can't find julia.h. Please input the path to the file.\")\n",
" path = readline()\n",
" while (!ispath(path) )\n",
" println(\"Can't find julia.h. Please input the path to the file.\")\n",
" path = readline()\n",
" end\n",
" path\n",
"end\n",
"C_code = raw\"\"\"\n",
"#include \"julia.h\"\n",
"\n",
"int jl_array_maxsize(jl_array_t *a) {\n",
" return a->maxsize;\n",
"}\n",
"\"\"\"\n",
"\n",
"const Clib = tempname()\n",
"open(`gcc -I $header -fPIC -O3 -msse3 -xc -shared -o $(Clib * \".\" * Libdl.dlext) -`, \"w\") do f\n",
" print(f, C_code) \n",
"end\n",
"arraycap(v) = ccall((:jl_array_maxsize, Clib), Int32, (Any,), v)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"10"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"v = rand(10)\n",
"arraycap(v)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The capacity is 0\n"
]
}
],
"source": [
"v = Vector{Int}(undef, 0)\n",
"println(\"The capacity is \", arraycap(v))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# of elements = 1: cacacity = 4\n",
"# of elements = 2: cacacity = 4\n",
"# of elements = 3: cacacity = 4\n",
"# of elements = 4: cacacity = 4\n",
"# of elements = 5: cacacity = 8\n",
"# of elements = 6: cacacity = 8\n",
"# of elements = 7: cacacity = 8\n",
"# of elements = 8: cacacity = 8\n",
"# of elements = 9: cacacity = 16\n",
"# of elements = 10: cacacity = 16\n",
"# of elements = 11: cacacity = 16\n",
"# of elements = 12: cacacity = 16\n",
"# of elements = 13: cacacity = 16\n",
"# of elements = 14: cacacity = 16\n",
"# of elements = 15: cacacity = 16\n",
"# of elements = 16: cacacity = 16\n",
"# of elements = 17: cacacity = 32\n",
"# of elements = 18: cacacity = 32\n",
"# of elements = 19: cacacity = 32\n",
"# of elements = 20: cacacity = 32\n",
"# of elements = 21: cacacity = 32\n",
"# of elements = 22: cacacity = 32\n",
"# of elements = 23: cacacity = 32\n",
"# of elements = 24: cacacity = 32\n",
"# of elements = 25: cacacity = 32\n",
"# of elements = 26: cacacity = 32\n",
"# of elements = 27: cacacity = 32\n",
"# of elements = 28: cacacity = 32\n",
"# of elements = 29: cacacity = 32\n",
"# of elements = 30: cacacity = 32\n",
"# of elements = 31: cacacity = 32\n",
"# of elements = 32: cacacity = 32\n"
]
}
],
"source": [
"for i in 1:32\n",
" push!(v, i)\n",
" println(\"# of elements = $(lpad(i, 3)): cacacity = $(arraycap(v))\")\n",
"end"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"capacity = 10000000\n"
]
}
],
"source": [
"sizehint!(v, 10_000_000)\n",
"println(\"capacity = \", arraycap(v))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"50000000"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"n = 50_000_000"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"capacity = 57944824\n"
]
}
],
"source": [
"v = Vector{Int}(undef, 0)\n",
"for i in 1:n\n",
" push!(v, i*10)\n",
"end\n",
"println(\"capacity = \", arraycap(v))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"25.78817646050732"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"log2(arraycap(v))"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/latex": [
"\\begin{tabular}\n",
"{l | r | l}\n",
"name & size & summary \\\\\n",
"\\hline\n",
"Base & & Module \\\\\n",
"C\\_code & 91 bytes & String \\\\\n",
"Clib & 24 bytes & String \\\\\n",
"Core & & Module \\\\\n",
"Main & & Module \\\\\n",
"arraycap & 0 bytes & typeof(arraycap) \\\\\n",
"header & 69 bytes & String \\\\\n",
"n & 8 bytes & Int64 \\\\\n",
"startupfile & 50 bytes & String \\\\\n",
"v & 381.470 MiB & 50000000-element Array\\{Int64,1\\} \\\\\n",
"\\end{tabular}\n"
],
"text/markdown": [
"| name | size | summary |\n",
"|:----------- | -----------:|:------------------------------- |\n",
"| Base | | Module |\n",
"| C_code | 91 bytes | String |\n",
"| Clib | 24 bytes | String |\n",
"| Core | | Module |\n",
"| Main | | Module |\n",
"| arraycap | 0 bytes | typeof(arraycap) |\n",
"| header | 69 bytes | String |\n",
"| n | 8 bytes | Int64 |\n",
"| startupfile | 50 bytes | String |\n",
"| v | 381.470 MiB | 50000000-element Array{Int64,1} |\n"
],
"text/plain": [
"name size summary \n",
"––––––––––– ––––––––––– –––––––––––––––––––––––––––––––\n",
"Base Module \n",
"C_code 91 bytes String \n",
"Clib 24 bytes String \n",
"Core Module \n",
"Main Module \n",
"arraycap 0 bytes typeof(arraycap) \n",
"header 69 bytes String \n",
"n 8 bytes Int64 \n",
"startupfile 50 bytes String \n",
"v 381.470 MiB 50000000-element Array{Int64,1}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"varinfo()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" total used free shared buff/cache available\n",
"Mem: 7852 4709 470 317 2672 2572\n",
"Swap: 4095 4 4091\n"
]
},
{
"data": {
"text/plain": [
"Process(`\u001b[4mfree\u001b[24m \u001b[4m-m\u001b[24m`, ProcessExited(0))"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"run(`free -m`)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"57944824\n"
]
}
],
"source": [
"for i in 1:n\n",
" pop!(v)\n",
"end\n",
"println(arraycap(v))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/latex": [
"\\begin{tabular}\n",
"{l | r | l}\n",
"name & size & summary \\\\\n",
"\\hline\n",
"Base & & Module \\\\\n",
"C\\_code & 91 bytes & String \\\\\n",
"Clib & 24 bytes & String \\\\\n",
"Core & & Module \\\\\n",
"Main & & Module \\\\\n",
"arraycap & 0 bytes & typeof(arraycap) \\\\\n",
"header & 69 bytes & String \\\\\n",
"n & 8 bytes & Int64 \\\\\n",
"startupfile & 50 bytes & String \\\\\n",
"v & 40 bytes & 0-element Array\\{Int64,1\\} \\\\\n",
"\\end{tabular}\n"
],
"text/markdown": [
"| name | size | summary |\n",
"|:----------- | --------:|:------------------------ |\n",
"| Base | | Module |\n",
"| C_code | 91 bytes | String |\n",
"| Clib | 24 bytes | String |\n",
"| Core | | Module |\n",
"| Main | | Module |\n",
"| arraycap | 0 bytes | typeof(arraycap) |\n",
"| header | 69 bytes | String |\n",
"| n | 8 bytes | Int64 |\n",
"| startupfile | 50 bytes | String |\n",
"| v | 40 bytes | 0-element Array{Int64,1} |\n"
],
"text/plain": [
"name size summary \n",
"––––––––––– –––––––– ––––––––––––––––––––––––\n",
"Base Module \n",
"C_code 91 bytes String \n",
"Clib 24 bytes String \n",
"Core Module \n",
"Main Module \n",
"arraycap 0 bytes typeof(arraycap) \n",
"header 69 bytes String \n",
"n 8 bytes Int64 \n",
"startupfile 50 bytes String \n",
"v 40 bytes 0-element Array{Int64,1}"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"varinfo()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" total used free shared buff/cache available\n",
"Mem: 7852 4713 467 316 2671 2569\n",
"Swap: 4095 4 4091\n"
]
},
{
"data": {
"text/plain": [
"Process(`\u001b[4mfree\u001b[24m \u001b[4m-m\u001b[24m`, ProcessExited(0))"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"run(`free -m`) # megabytes"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10\n",
"20\n",
"30\n",
"40\n",
"50\n",
"60\n",
"70\n",
"80\n",
"90\n",
"100\n"
]
}
],
"source": [
"for i in 1:10\n",
" println(unsafe_load(pointer(v), i))\n",
"end"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" total used free shared buff/cache available\n",
"Mem: 7852 4316 859 320 2676 2962\n",
"Swap: 4095 4 4091\n"
]
},
{
"data": {
"text/plain": [
"Process(`\u001b[4mfree\u001b[24m \u001b[4m-m\u001b[24m`, ProcessExited(0))"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"v = nothing\n",
"GC.gc()\n",
"run(`free -m`)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"setarraypar (generic function with 1 method)"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"C_code2 = raw\"\"\"\n",
"#include \"julia.h\"\n",
"\n",
"void jl_set_array(jl_array_t *a, int n) {\n",
" a->nrows = n;\n",
" a->length = n;\n",
" a->maxsize = n;\n",
" a->offset = 0;\n",
"\n",
" return;\n",
"}\n",
"\"\"\"\n",
"\n",
"const Clib2 = tempname()\n",
"open(`gcc -I $header -fPIC -O3 -msse3 -xc -shared -o $(Clib2 * \".\" * Libdl.dlext) -`, \"w\") do f\n",
" print(f, C_code2) \n",
"end\n",
"setarraypar(v, n) = ccall((:jl_set_array, Clib2), Nothing, (Any,Int32,), v, n)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"50000000-element Array{Int64,1}:\n",
" 10\n",
" 20\n",
" 30\n",
" 40\n",
" 50\n",
" 60\n",
" 70\n",
" 80\n",
" 90\n",
" 100\n",
" 110\n",
" 120\n",
" 130\n",
" ⋮\n",
" 499999890\n",
" 499999900\n",
" 499999910\n",
" 499999920\n",
" 499999930\n",
" 499999940\n",
" 499999950\n",
" 499999960\n",
" 499999970\n",
" 499999980\n",
" 499999990\n",
" 500000000"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"v = Vector{Int}(undef, 0)\n",
"n = 50_000_000\n",
"for i in 1:n\n",
" push!(v, i*10)\n",
"end\n",
"v"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0-element Array{Int64,1}"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"for i in 1:n\n",
" pop!(v)\n",
"end\n",
"v"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"50000000-element Array{Int64,1}:\n",
" 10\n",
" 20\n",
" 30\n",
" 40\n",
" 50\n",
" 60\n",
" 70\n",
" 80\n",
" 90\n",
" 100\n",
" 110\n",
" 120\n",
" 130\n",
" ⋮\n",
" 499999890\n",
" 499999900\n",
" 499999910\n",
" 499999920\n",
" 499999930\n",
" 499999940\n",
" 499999950\n",
" 499999960\n",
" 499999970\n",
" 499999980\n",
" 499999990\n",
" 500000000"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"setarraypar(v, Int32(n))\n",
"v"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Julia 1.1.0",
"language": "julia",
"name": "julia-1.1"
},
"language_info": {
"file_extension": ".jl",
"mimetype": "application/julia",
"name": "julia",
"version": "1.1.0"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment